Coverage for /home/runner/work/viur-core/viur-core/viur/src/viur/core/db/query.py: 33%

1from __future__ import annotations 

2 

3import base64 

4import copy 

5import functools 

6import logging 

7import typing as t 

8 

9from viur.core.config import conf 

10from .transport import count, get, run_single_filter 

11from .types import ( 

12 DATASTORE_BASE_TYPES, 

13 Entity, 

14 KEY_SPECIAL_PROPERTY, 

15 QueryDefinition, 

16 QueryOrder, 

17 SortOrder, 

18 TFilters, 

19 TOrders, 

20 TOrFilters, 

21 Key 

22) 

23from . import utils 

24 

25if t.TYPE_CHECKING: 25 ↛ 26line 25 didn't jump to line 26 because the condition on line 25 was never true

26 from viur.core.skeleton import SkeletonInstance, SkelList 

27 

28TOrderHook = t.TypeVar("TOrderHook", bound=t.Callable[["Query", TOrders], TOrders]) 

29TFilterHook = t.TypeVar("TFilterHook", bound=t.Callable[ 

30 ["Query", str, DATASTORE_BASE_TYPES | list[DATASTORE_BASE_TYPES]], TFilters 

31]) 

32 

33 

34def _entryMatchesQuery( 

35 entry: Entity, 

36 singleFilter: dict, 

37 or_filters: TOrFilters | None = None, 

38) -> bool: 

39 """ 

40 Utility function which checks if the given entity could have been returned by a query filtering by the 

41 properties in singleFilter. This can be used if a list of entities have been retrieved (e.g. by a 3rd party 

42 full text search engine) and these have now to be checked against the filter returned by their modules 

43 :meth:`viur.core.prototypes.list.listFilter` method. 

44 :param entry: The entity which will be tested 

45 :param singleFilter: A dictionary containing all the filters from the query 

46 :param or_filters: Optional list of OR groups; each group is a list of (filterStr, value) pairs 

47 :return: True if the entity could have been returned by such an query, False otherwise 

48 """ 

49 

50 def doesMatch(entryValue: t.Any, requestedValue: t.Any, opcode: str) -> bool: 

51 if isinstance(entryValue, list): 

52 return any([doesMatch(x, requestedValue, opcode) for x in entryValue]) 

53 if opcode == "=" and entryValue == requestedValue: 

54 return True 

55 elif opcode == "<" and entryValue < requestedValue: 55 ↛ 56line 55 didn't jump to line 56 because the condition on line 55 was never true

56 return True 

57 elif opcode == ">" and entryValue > requestedValue: 

58 return True 

59 elif opcode == "<=" and entryValue <= requestedValue: 59 ↛ 60line 59 didn't jump to line 60 because the condition on line 59 was never true

60 return True 

61 elif opcode == ">=" and entryValue >= requestedValue: 61 ↛ 62line 61 didn't jump to line 62 because the condition on line 61 was never true

62 return True 

63 elif opcode == "IN" and entryValue in requestedValue: 

64 return True 

65 elif opcode == "NOT_IN" and entryValue not in requestedValue: 

66 return True 

67 # any()-semantics for multi-value properties: list dispatch above handles iteration 

68 elif opcode == "!=" and entryValue != requestedValue: 

69 return True 

70 return False 

71 

72 for filterStr, filterValue in singleFilter.items(): 

73 field, opcode = filterStr.split(" ") 

74 entryValue = entry.get(field) 

75 if not doesMatch(entryValue, filterValue, opcode): 

76 return False 

77 

78 if or_filters: 

79 for or_group in or_filters: 

80 if not any( 

81 doesMatch(entry.get(fs.split(" ", 1)[0]), v, fs.split(" ", 1)[1]) 

82 for fs, v in or_group 

83 ): 

84 return False 

85 

86 return True 

87 

88 

89class Query(object): 

90 """ 

91 Base Class for querying the datastore. Its API is similar to the google.cloud.datastore.query API, 

92 but it provides the necessary hooks for relational or random queries, the fulltext search as well as support 

93 for IN filters. 

94 """ 

95 

96 def __init__(self, kind: str, srcSkelClass: t.Union["SkeletonInstance", None] = None, *args, **kwargs): 

97 """ 

98 Constructs a new Query. 

99 :param kind: The kind to run this query on. This may be later overridden to run on a different kind (like 

100 viur-relations), but it's guaranteed to return only entities of that kind. 

101 :param srcSkelClass: If set, enables data-model depended queries (like relational queries) as well as the 

102 :meth:fetch method 

103 """ 

104 super().__init__() 

105 self.kind = kind 

106 self.srcSkel = srcSkelClass 

107 self.queries: t.Union[None, QueryDefinition, t.List[QueryDefinition]] = QueryDefinition(kind, {}, []) 

108 self._filterHook: TFilterHook | None = None 

109 self._orderHook: TOrderHook | None = None 

110 # Sometimes, the default merge functionality from MultiQuery is not sufficient 

111 self._customMultiQueryMerge: t.Union[None, t.Callable[[Query, t.List[t.List[Entity]], int], t.List[Entity]]] \ 

112 = None 

113 # Some (Multi-)Queries need a different amount of results per subQuery than actually returned 

114 self._calculateInternalMultiQueryLimit: t.Union[None, t.Callable[[Query, int], int]] = None 

115 # Allow carrying custom data along with the query. 

116 # Currently only used by SpatialBone to record the guaranteed correctness 

117 self.customQueryInfo = {} 

118 self.origKind = kind 

119 self._lastEntry = None 

120 self._fulltextQueryString: t.Union[None, str] = None 

121 self.lastCursor = None 

122 # if not kind.startswith("viur") and not kwargs.get("_excludeFromAccessLog"): 

123 # accessLog = current_db_access_log.get() 

124 # if isinstance(accessLog, set): 

125 # accessLog.add(kind) 

126 

127 def setFilterHook(self, hook: TFilterHook) -> TFilterHook | None: 

128 """ 

129 Installs *hook* as a callback function for new filters. 

130 

131 *hook* will be called each time a new filter constrain is added to the query. 

132 This allows e.g. the relationalBone to rewrite constrains added after the initial 

133 processing of the query has been done (e.g. by ``listFilter()`` methods). 

134 

135 :param hook: The function to register as callback. 

136 A value of None removes the currently active hook. 

137 :returns: The previously registered hook (if any), or None. 

138 """ 

139 old = self._filterHook 

140 self._filterHook = hook 

141 return old 

142 

143 def setOrderHook(self, hook: TOrderHook) -> TOrderHook | None: 

144 """ 

145 Installs *hook* as a callback function for new orderings. 

146 

147 *hook* will be called each time a :func:`db.Query.order` is called on this query. 

148 

149 :param hook: The function to register as callback. 

150 A value of None removes the currently active hook. 

151 :returns: The previously registered hook (if any), or None. 

152 """ 

153 old = self._orderHook 

154 self._orderHook = hook 

155 return old 

156 

157 def mergeExternalFilter(self, filters: dict) -> t.Self: 

158 """ 

159 Safely merges filters according to the data model. 

160 

161 Its only valid to call this function if the query has been created using 

162 :func:`core.skeleton.Skeleton.all`. 

163 

164 It's safe to pass filters received from an external source (a user); 

165 unknown/invalid filters will be ignored, so the query-object is kept in a 

166 valid state even when processing malformed data. 

167 

168 If complex queries are needed (e.g. filter by relations), this function 

169 shall also be used. 

170 

171 See also :meth:`filter` for simple filters. 

172 

173 :param filters: A dictionary of attributes and filter pairs. 

174 :returns: Returns the query itself for chaining. 

175 """ 

176 if self.srcSkel is None: 

177 raise NotImplementedError("This query has not been created using skel.all()") 

178 

179 if self.queries is None: # This query is already unsatisfiable and adding more constraints won't change this 

180 return self 

181 

182 skel = self.srcSkel 

183 

184 if "search" in filters: 

185 if self.srcSkel.customDatabaseAdapter and self.srcSkel.customDatabaseAdapter.providesFulltextSearch: 

186 self._fulltextQueryString = str(filters["search"]) 

187 else: 

188 logging.warning( 

189 "Got a fulltext search query for %s which does not have a suitable customDatabaseAdapter" 

190 % self.srcSkel.kindName 

191 ) 

192 self.queries = None 

193 

194 bones = [(y, x) for x, y in skel.items()] 

195 

196 try: 

197 # Process filters first 

198 for bone, key in bones: 

199 bone.buildDBFilter(key, skel, self, filters) 

200 

201 # Parse orders 

202 for bone, key in bones: 

203 bone.buildDBSort(key, skel, self, filters) 

204 

205 except RuntimeError as e: 

206 logging.exception(e) 

207 self.queries = None 

208 return self 

209 

210 startCursor = endCursor = None 

211 

212 if "cursor" in filters and filters["cursor"] and filters["cursor"].lower() != "none": 

213 startCursor = filters["cursor"] 

214 

215 if "endcursor" in filters and filters["endcursor"] and filters["endcursor"].lower() != "none": 

216 endCursor = filters["endcursor"] 

217 

218 if startCursor or endCursor: 

219 self.setCursor(startCursor, endCursor) 

220 

221 if limit := filters.get("limit"): 

222 try: 

223 limit = int(limit) 

224 

225 # disallow limit beyond conf.db.query_external_limit 

226 if limit > conf.db.query_external_limit: 

227 limit = conf.db.query_external_limit 

228 

229 # forbid any limit < 0, which might bypass defaults 

230 if limit < 0: 

231 limit = 0 

232 

233 self.limit(limit) 

234 except ValueError: 

235 pass # ignore this 

236 

237 return self 

238 

239 def filter(self, prop: str, value: DATASTORE_BASE_TYPES | list[DATASTORE_BASE_TYPES]) -> t.Self: 

240 """ 

241 Adds a new constraint to this query. 

242 

243 See also :meth:`mergeExternalFilter` for a safer filter implementation. 

244 

245 :param prop: Name of the property + operation we'll filter by 

246 :param value: The value of that filter. 

247 :returns: Returns the query itself for chaining. 

248 """ 

249 if self.queries is None: 249 ↛ 251line 249 didn't jump to line 251 because the condition on line 249 was never true

250 # This query is already unsatisfiable and adding more constrains to this won't change this 

251 return self 

252 if self._filterHook is not None: 252 ↛ 253line 252 didn't jump to line 253 because the condition on line 252 was never true

253 try: 

254 r = self._filterHook(self, prop, value) 

255 except RuntimeError: 

256 self.queries = None 

257 return self 

258 if r is None: 

259 # The Hook did something special directly on 'self' to apply that filter, 

260 # no need for us to do anything 

261 return self 

262 prop, value = r 

263 if " " not in prop: 263 ↛ 264line 263 didn't jump to line 264 because the condition on line 263 was never true

264 field = prop 

265 op = "=" 

266 else: 

267 field, op = prop.split(" ") 

268 

269 # Normalize to uppercase for native Datastore operators passed as lowercase 

270 op = op.upper() if op.upper() in {"IN", "NOT_IN"} else op 

271 

272 if op in {"IN", "!=", "NOT_IN"} and not isinstance(self.queries, list): 

273 if f"{field} {op}" in self.queries.filters: 273 ↛ 274line 273 didn't jump to line 274 because the condition on line 273 was never true

274 raise ValueError(f"Cannot use multiple {op} filters on the same field '{field}'") 

275 

276 filterStr = f"{field} {op}" 

277 if isinstance(self.queries, list): 277 ↛ 278line 277 didn't jump to line 278 because the condition on line 277 was never true

278 for singleFilter in self.queries: 

279 if filterStr not in singleFilter.filters: 

280 singleFilter.filters[filterStr] = value 

281 else: 

282 if not isinstance(singleFilter.filters[filterStr], list): 

283 singleFilter.filters[filterStr] = [singleFilter.filters[filterStr]] 

284 singleFilter.filters[filterStr].append(value) 

285 else: 

286 if filterStr not in self.queries.filters: 286 ↛ 289line 286 didn't jump to line 289 because the condition on line 286 was always true

287 self.queries.filters[filterStr] = value 

288 else: 

289 if not isinstance(self.queries.filters[filterStr], list): 

290 self.queries.filters[filterStr] = [self.queries.filters[filterStr]] 

291 self.queries.filters[filterStr].append(value) 

292 

293 if op in {"<", "<=", ">", ">="}: 293 ↛ 294line 293 didn't jump to line 294 because the condition on line 293 was never true

294 if isinstance(self.queries, list): 

295 for queryObj in self.queries: 

296 if not queryObj.orders or queryObj.orders[0][0] != field: 

297 queryObj.orders = [QueryOrder(field)] + (queryObj.orders or []) 

298 else: 

299 if not self.queries.orders or self.queries.orders[0][0] != field: 

300 self.queries.orders = [QueryOrder(field)] + (self.queries.orders or []) 

301 return self 

302 

303 def or_filter(self, *conditions: tuple[str, DATASTORE_BASE_TYPES]) -> t.Self: 

304 """ 

305 Add an OR composite filter group. 

306 

307 Each call appends one OR group; multiple calls produce multiple groups 

308 that are AND-ed together with each other and with any regular filters. 

309 

310 Example — continent is Africa OR Asia:: 

311 

312 q.or_filter(("continent =", "Africa"), ("continent =", "Asia")) 

313 

314 Example — two independent OR groups (both must match):: 

315 

316 q.or_filter(("continent =", "Africa"), ("continent =", "Asia")) 

317 q.or_filter(("sortindex >", 200), ("sortindex <", 50)) 

318 

319 :param conditions: One or more ``("field op", value)`` pairs to OR together. 

320 :returns: Returns the query itself for chaining. 

321 """ 

322 if self.queries is None: 322 ↛ 323line 322 didn't jump to line 323 because the condition on line 322 was never true

323 return self 

324 

325 parsed = [] 

326 for prop, value in conditions: 

327 if " " not in prop: 

328 field, op = prop, "=" 

329 else: 

330 field, op = prop.split(" ", 1) 

331 op = op.upper() if op.upper() in {"IN", "NOT_IN"} else op 

332 parsed.append((f"{field} {op}", value)) 

333 

334 if isinstance(self.queries, list): 334 ↛ 335line 334 didn't jump to line 335 because the condition on line 334 was never true

335 for q in self.queries: 

336 q.or_filters.append(parsed) 

337 else: 

338 self.queries.or_filters.append(parsed) 

339 return self 

340 

341 def order(self, *orderings: QueryOrder | t.Tuple[str, SortOrder] | str) -> t.Self: 

342 """ 

343 Specify a query sorting. 

344 

345 Resulting entities will be sorted by the first property argument, then by the 

346 second, and so on. 

347 

348 The following example 

349 

350 .. code-block:: python 

351 

352 query = Query("Person") 

353 query.order( 

354 db.QueryOrder("bday"), 

355 db.QueryOrder("age", db.SortOrder.Descending), 

356 ) 

357 

358 sorts every Person in order of their birthday, starting with January 1. 

359 People with the same birthday are sorted by age, oldest to youngest. 

360 

361 

362 ``order()`` may be called multiple times. Each call resets the sort order 

363 from scratch. 

364 

365 If an inequality filter exists in this Query it must be the first property 

366 passed to ``order()``. Any number of sort orders may be used after the 

367 inequality filter property. Without inequality filters, any number of 

368 filters with different orders may be specified. 

369 

370 Entities with multiple values for an order property are sorted by their 

371 lowest value. 

372 

373 Note that a sort order implies an existence filter! In other words, 

374 Entities without the sort order property are filtered out, and *not* 

375 included in the query results. 

376 

377 If the sort order property has different types in different entities - 

378 e.g. if bob["id"] is an int and fred["id"] is a string - the entities will be 

379 grouped first by the property type, then sorted within type. No attempt is 

380 made to compare property values across types. 

381 

382 

383 :param orderings: The properties to sort by, in sort order. Each argument may be a 

384 :class:`QueryOrder`, a ``(name, direction)`` tuple, or a plain ``str`` (implies 

385 ``SortOrder.Ascending``). 

386 :returns: Returns the query itself for chaining. 

387 """ 

388 if self.queries is None: 388 ↛ 390line 388 didn't jump to line 390 because the condition on line 388 was never true

389 # This Query is unsatisfiable - don't try to bother 

390 return self 

391 

392 # Check for correct order subscript 

393 orders = [] 

394 for order in orderings: 

395 if isinstance(order, str): 

396 order = QueryOrder(order) 

397 elif isinstance(order, QueryOrder): 397 ↛ 398line 397 didn't jump to line 398 because the condition on line 397 was never true

398 pass 

399 elif ( 399 ↛ 406line 399 didn't jump to line 406 because the condition on line 399 was always true

400 isinstance(order, (tuple, list)) and 

401 len(order) == 2 and 

402 isinstance(order[0], str) and isinstance(order[1], SortOrder) 

403 ): 

404 order = QueryOrder(order[0], order[1]) 

405 else: 

406 raise TypeError( 

407 f"Invalid ordering {order!r}, expected a (str, SortOrder) tuple or QueryOrder." 

408 f' Try: `QueryOrder("{order}")`' 

409 ) 

410 orders.append(order) 

411 

412 if self._orderHook is not None: 412 ↛ 413line 412 didn't jump to line 413 because the condition on line 412 was never true

413 try: 

414 orders = self._orderHook(self, orders) 

415 except RuntimeError: 

416 self.queries = None 

417 return self 

418 if orders is None: 

419 return self 

420 

421 if isinstance(self.queries, list): 421 ↛ 422line 421 didn't jump to line 422 because the condition on line 421 was never true

422 for query in self.queries: 

423 query.orders = list(orders) 

424 else: 

425 self.queries.orders = list(orders) 

426 

427 return self 

428 

429 def setCursor(self, startCursor: str, endCursor: t.Optional[str] = None) -> t.Self: 

430 """ 

431 Sets the start and optionally end cursor for this query. 

432 

433 The result set will only include results between these cursors. 

434 The cursor is generated by an earlier query with exactly the same configuration. 

435 

436 It's safe to use client-supplied cursors, a cursor can't be abused to access entities 

437 which don't match the current filters. 

438 

439 :param startCursor: The start cursor for this query. 

440 :param endCursor: The end cursor for this query. 

441 :returns: Returns the query itself for chaining. 

442 """ 

443 if isinstance(self.queries, list): 

444 for query in self.queries: 

445 assert isinstance(query, QueryDefinition) 

446 if startCursor: 

447 query.startCursor = base64.urlsafe_b64decode(startCursor.encode("ASCII")).decode("ASCII") 

448 if endCursor: 

449 query.endCursor = base64.urlsafe_b64decode(endCursor.encode("ASCII")).decode("ASCII") 

450 else: 

451 assert isinstance(self.queries, QueryDefinition) 

452 if startCursor: 

453 self.queries.startCursor = base64.urlsafe_b64decode(startCursor.encode("ASCII")).decode("ASCII") 

454 if endCursor: 

455 self.queries.endCursor = base64.urlsafe_b64decode(endCursor.encode("ASCII")).decode("ASCII") 

456 return self 

457 

458 def limit(self, limit: int) -> t.Self: 

459 """ 

460 Sets the query limit to *limit* entities in the result. 

461 

462 :param limit: The maximum number of entities per batch. 

463 :returns: Returns the query itself for chaining. 

464 """ 

465 if isinstance(self.queries, QueryDefinition): 

466 self.queries.limit = limit 

467 elif isinstance(self.queries, list): 

468 for query in self.queries: 

469 query.limit = limit 

470 

471 return self 

472 

473 def distinctOn(self, keyList: t.List[str]) -> t.Self: 

474 """ 

475 Ensure only entities with distinct values on the fields listed are returned. 

476 This will implicitly override your SortOrder as all fields listed in keyList have to be sorted first. 

477 """ 

478 if isinstance(self.queries, QueryDefinition): 

479 self.queries.distinct = keyList 

480 elif isinstance(self.queries, list): 

481 for query in self.queries: 

482 query.distinct = keyList 

483 return self 

484 

485 def getCursor(self) -> t.Optional[str]: 

486 """ 

487 Get a valid cursor from the last run of this query. 

488 

489 The source of this cursor varies depending on what the last call was: 

490 - :meth:`run`: A cursor that points immediately behind the 

491 last result pulled off the returned iterator. 

492 - :meth:`get`: A cursor that points immediately behind the 

493 last result in the returned list. 

494 

495 :returns: A cursor that can be used in subsequent query requests or None if that query does not support 

496 cursors or there are no more elements to fetch 

497 """ 

498 if isinstance(self.queries, QueryDefinition): 

499 q = self.queries 

500 elif isinstance(self.queries, list): 

501 for query in self.queries: 

502 if query.currentCursor: 

503 q = query 

504 break 

505 else: 

506 q = self.queries[0] 

507 return base64.urlsafe_b64encode(q.currentCursor).decode("ASCII") if q.currentCursor else None 

508 

509 def get_orders(self) -> t.List[QueryOrder] | None: 

510 """ 

511 Get the orders from this query. 

512 

513 :returns: The orders form this query as a list if there is no orders set it returns None 

514 """ 

515 q = self.queries 

516 

517 if isinstance(q, (list, tuple)): 517 ↛ 518line 517 didn't jump to line 518 because the condition on line 517 was never true

518 q = q[0] 

519 

520 if not isinstance(q, QueryDefinition): 520 ↛ 521line 520 didn't jump to line 521 because the condition on line 520 was never true

521 raise ValueError( 

522 f"self.queries can only be a 'QueryDefinition' or a list of, but found {self.queries!r}" 

523 ) 

524 

525 return q.orders or None 

526 

527 # TODO We need this the kind is already public. 

528 def getKind(self) -> str: 

529 """ 

530 :returns: the *current* kind of this query. 

531 This may not be the kind this query has been constructed with 

532 as relational bones may rewrite this. 

533 """ 

534 return self.kind 

535 

536 def _run_single_filter_query(self, query: QueryDefinition, limit: int, keys_only: bool) -> t.List[Entity]: 

537 """ 

538 Internal helper function that runs a single query definition on the datastore and returns a list of 

539 entities found. 

540 :param query: The querydefinition (filters, orders, distinct etc.) to run against the datastore 

541 :param limit: How many results should at most be returned 

542 :return: The first *limit* entities that matches this query 

543 """ 

544 return run_single_filter(query, limit, keys_only) 

545 

546 def _merge_multi_query_results(self, input_result: t.List[t.List[Entity]]) -> t.List[Entity]: 

547 """ 

548 Merge the lists of entries into a single list; removing duplicates and restoring sort-order 

549 :param input_result: Nested Lists of Entries returned by each individual query run 

550 :return: Sorted & deduplicated list of entries 

551 """ 

552 seen_keys = set() 

553 res = [] 

554 for subList in input_result: 

555 for entry in subList: 

556 key = entry.key 

557 if key in seen_keys: 

558 continue 

559 seen_keys.add(key) 

560 res.append(entry) 

561 # FIXME: What about filters that mix different inequality filters? 

562 # Currently, we'll now simply ignore any implicit sortorder. 

563 return self._resort_result(res, {}, self.queries[0].orders) 

564 

565 def _resort_result( 

566 self, 

567 entities: t.List[Entity], 

568 filters: t.Dict[str, DATASTORE_BASE_TYPES], 

569 orders: t.List[QueryOrder], 

570 ) -> t.List[Entity]: 

571 """ 

572 Internal helper that takes a (deduplicated) list of entities that has been fetched from different internal 

573 queries (e.g. from SpatialBone or RandomSliceBone custom multi-queries) and resorts the list so it matches 

574 the query again. Regular IN/!= filters no longer use this path — they are handled natively by the Datastore. 

575 

576 :param entities: t.List of entities to resort 

577 :param filters: The filter used in the query (used to determine implicit sort order by an inequality filter) 

578 :param orders: The sort-orders to apply 

579 :return: The sorted list 

580 """ 

581 

582 def getVal(src: Entity, fieldVars: t.Union[str, t.Tuple[str]], direction: SortOrder) -> t.Any: 

583 # Descent into the target until we reach the property we're looking for 

584 if isinstance(fieldVars, tuple): 

585 for fv in fieldVars: 

586 if fv not in src: 

587 return None 

588 src = src[fv] 

589 else: 

590 if fieldVars not in src: 

591 return (str(type(None)), 0) 

592 src = src[fieldVars] 

593 # Lists are handled differently, here the smallest or largest value determines it's position in the result 

594 if isinstance(src, list) and len(src): 

595 try: 

596 src.sort() 

597 except TypeError: 

598 # It's a list of dicts or the like for which no useful sort-order is specified 

599 pass 

600 if direction == SortOrder.Ascending: 

601 src = src[0] 

602 else: 

603 src = src[-1] 

604 # We must return this tuple because inter-type comparison isn't possible in Python3 anymore 

605 return str(type(src)), src if src is not None else 0 

606 

607 # Check if we have an inequality filter which implies a sortorder 

608 ineqFilter = None 

609 for k, _ in filters.items(): 

610 end = k[-2:] 

611 if "<" in end or ">" in end: 

612 ineqFilter = k.split(" ")[0] 

613 break 

614 if ineqFilter and (not orders or not orders[0].name == ineqFilter): 

615 orders = [QueryOrder(ineqFilter)] + (orders or []) 

616 

617 for orderField, direction in orders[::-1]: 

618 if orderField == KEY_SPECIAL_PROPERTY: 

619 pass # FIXME !! 

620 # entities.sort(key=lambda x: x.key, reverse=direction == SortOrder.Descending) 

621 else: 

622 try: 

623 entities.sort(key=functools.partial(getVal, fieldVars=orderField, direction=direction), 

624 reverse=direction == SortOrder.Descending) 

625 except TypeError: 

626 # We hit some incomparable types 

627 pass 

628 return entities 

629 

630 def _fixKind(self, resultList: t.List[Entity]) -> t.List[Entity]: 

631 """ 

632 Jump to parentKind if necessary (used in relations) 

633 """ 

634 resultList = list(resultList) 

635 if ( 

636 resultList 

637 and resultList[0].key.kind != self.origKind 

638 and resultList[0].key.parent 

639 and resultList[0].key.parent.kind == self.origKind 

640 ): 

641 return list(get(list(dict.fromkeys([x.key.parent for x in resultList])))) 

642 

643 return resultList 

644 

645 def run(self, limit: int = -1, keys_only: bool = False) -> t.List[Entity | Key]: 

646 """ 

647 Run this query. 

648 

649 It is more efficient to use *limit* if the number of results is known. 

650 

651 If queried data is wanted as instances of Skeletons, :meth:`fetch` 

652 should be used. 

653 

654 :param limit: Limits the query to the defined maximum entities. 

655 :param keys_only: If True, only return entities keys. 

656 

657 :returns: The list of found entities 

658 

659 :raises: :exc:`BadFilterError` if a filter string is invalid 

660 :raises: :exc:`BadValueError` if a filter value is invalid. 

661 """ 

662 if self.queries is None: 

663 if conf.debug.trace_queries: 

664 logging.debug(f"Query on {self.kind} aborted as being not satisfiable") 

665 return [] 

666 

667 if self._fulltextQueryString: 

668 if utils.is_in_transaction(): 

669 raise ValueError("Can't run fulltextSearch inside transactions!") # InvalidStateError FIXME! 

670 if keys_only: 

671 raise ValueError("Can't run fulltextSearch with keysOnly!") 

672 qryStr = self._fulltextQueryString 

673 self._fulltextQueryString = None # Reset, so the adapter can still work with this query 

674 res = self.srcSkel.customDatabaseAdapter.fulltextSearch(qryStr, self) 

675 

676 if not self.srcSkel.customDatabaseAdapter.fulltextSearchGuaranteesQueryConstrains: 

677 # Search might yield results that are not included in the listfilter 

678 if isinstance(self.queries, QueryDefinition): # Just one 

679 res = [x for x in res if _entryMatchesQuery(x, self.queries.filters, self.queries.or_filters)] 

680 else: # Multi-Query, must match at least one 

681 res = [x for x in res if 

682 any([_entryMatchesQuery(x, y.filters, y.or_filters) for y in self.queries])] 

683 

684 elif isinstance(self.queries, list): 

685 limit = limit if limit >= 0 else self.queries[0].limit 

686 

687 # We have more than one query to run 

688 if self._calculateInternalMultiQueryLimit: 

689 limit = self._calculateInternalMultiQueryLimit(self, limit) 

690 

691 res = [] 

692 # We run all queries first (preventing multiple round-trips to the server) 

693 for singleQuery in self.queries: 

694 res.append(self._run_single_filter_query(singleQuery, limit, keys_only)) 

695 

696 # Wait for the actual results to arrive and convert the protobuffs to Entries 

697 res = [self._fixKind(x) for x in res] 

698 if self._customMultiQueryMerge: 

699 # We have a custom merge function, use that 

700 res = self._customMultiQueryMerge(self, res, limit) 

701 else: 

702 # We must merge (and sort) the results ourself 

703 res = self._merge_multi_query_results(res) 

704 

705 else: # We have just one single query 

706 res = self._fixKind(self._run_single_filter_query( 

707 self.queries, 

708 limit if limit >= 0 else self.queries.limit, 

709 keys_only 

710 )) 

711 

712 if res: 

713 if keys_only: 

714 res = [ 

715 obj if isinstance(obj, Key) else obj.key 

716 for obj in res 

717 if isinstance(obj, (Entity, Key)) 

718 ] 

719 self._lastEntry = res[-1] 

720 

721 return res 

722 

723 def count(self, up_to: int = 2 ** 63 - 1) -> int: 

724 """ 

725 The count operation cost one entity read for up to 1,000 index entries matched 

726 (https://cloud.google.com/datastore/docs/aggregation-queries#pricing) 

727 :param up_to can be sigend int 64 bit (max positive 2^31-1) 

728 

729 :returns: Count entries for this query. 

730 """ 

731 if self.queries is None: 

732 if conf.debug.trace_queries: 

733 logging.debug(f"Query on {self.kind} aborted as being not satisfiable") 

734 return -1 

735 elif isinstance(self.queries, list): 

736 raise ValueError("No count on Multiqueries") 

737 else: 

738 return count(queryDefinition=self.queries, up_to=up_to) 

739 

740 def fetch(self, limit: int = -1) -> "SkelList": 

741 """ 

742 Run this query and fetch results as :class:`core.skeleton.SkelList`. 

743 

744 This function is similar to :meth:`run`, but returns a 

745 :class:`core.skeleton.SkelList` instance instead of Entities. 

746 

747 :warning: The query must be limited! 

748 

749 If queried data is wanted as instances of Entity, :meth:`run` 

750 should be used. 

751 

752 :param limit: Limits the query to the defined maximum entities. 

753 

754 :raises: :exc:`BadFilterError` if a filter string is invalid 

755 :raises: :exc:`BadValueError` if a filter value is invalid. 

756 """ 

757 from viur.core.skeleton import SkelList, SkeletonInstance 

758 

759 if self.srcSkel is None: 

760 raise NotImplementedError("This query has not been created using skel.all()") 

761 

762 res = SkelList(self.srcSkel) 

763 

764 # FIXME: Why is this not like in ViUR2? 

765 for entity in self.run(limit): 

766 skel_instance = SkeletonInstance(self.srcSkel.skeletonCls, bone_map=self.srcSkel.boneMap) 

767 skel_instance.dbEntity = entity 

768 res.append(skel_instance) 

769 

770 res.getCursor = lambda: self.getCursor() 

771 res.get_orders = lambda: self.get_orders() 

772 

773 return res 

774 

775 def iter(self, keys_only=False) -> t.Iterator[Entity]: 

776 """ 

777 Run this query and return an iterator for the results. 

778 

779 The advantage of this function is, that it allows for iterating 

780 over a large result-set, as it hasn't have to be pulled in advance 

781 from the datastore. 

782 

783 This function intentionally ignores a limit set by :meth:`limit`. 

784 

785 :warning: If iterating over a large result set, make sure the query supports cursors. \ 

786 Otherwise, it might not return all results as the AppEngine doesn't maintain the view \ 

787 for a query for more than ~30 seconds. 

788 """ 

789 if self.queries is None: # Noting to pull here 

790 return 

791 elif isinstance(self.queries, list): 791 ↛ 792line 791 didn't jump to line 792 because the condition on line 791 was never true

792 raise ValueError("No iter on Multiqueries") 

793 while True: 

794 yield from self._run_single_filter_query(self.queries, 100, keys_only) 

795 if not self.queries.currentCursor: # We reached the end of that query 

796 break 

797 self.queries.startCursor = self.queries.currentCursor 

798 

799 def iter_skel(self) -> t.Iterator["SkeletonInstance"]: 

800 """ 

801 Run this query and return an iterator yielding :class:`core.skeleton.SkeletonInstance`. 

802 

803 This function is to :meth:`iter` what :meth:`fetch` is to :meth:`run`: it allows for 

804 iterating over a large result-set without pulling it from the datastore in advance, 

805 but yields SkeletonInstances instead of Entities. 

806 

807 It's only possible to use this function if this query has been created using 

808 :func:`core.skeleton.Skeleton.all`. 

809 

810 Every result is a separate SkeletonInstance which shares the bone-map of the 

811 source-skeleton, therefore collecting the results in a list or writing them 

812 within the loop behaves as expected. 

813 

814 This function intentionally ignores a limit set by :meth:`limit`. 

815 

816 :warning: If iterating over a large result set, make sure the query supports cursors. \ 

817 Otherwise, it might not return all results as the AppEngine doesn't maintain the view \ 

818 for a query for more than ~30 seconds. 

819 

820 :raises NotImplementedError: If this query has not been created using skel.all(). 

821 :raises ValueError: If this is a multi-query, which cannot be iterated. 

822 """ 

823 if self.srcSkel is None: 

824 raise NotImplementedError("This query has not been created using skel.all()") 

825 elif isinstance(self.queries, list): 

826 raise ValueError("No iter_skel on Multiqueries") 

827 

828 from viur.core.skeleton import SkeletonInstance 

829 

830 # Wrapped in an inner generator, so the checks above are raised on call and not on first next() 

831 def _iterate() -> t.Iterator["SkeletonInstance"]: 

832 for entity in self.iter(): 

833 skel_instance = SkeletonInstance(self.srcSkel.skeletonCls, bone_map=self.srcSkel.boneMap) 

834 skel_instance.dbEntity = entity 

835 yield skel_instance 

836 

837 return _iterate() 

838 

839 def getEntry(self) -> t.Union[None, Entity]: 

840 """