yjdata 0.1.5__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. yjdata-0.2.0/PKG-INFO +111 -0
  2. yjdata-0.2.0/README.md +100 -0
  3. {yjdata-0.1.5 → yjdata-0.2.0}/pyproject.toml +1 -1
  4. {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/dataset/common.py +5 -0
  5. {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/db/adapter.py +100 -8
  6. {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/db/sql.py +122 -4
  7. {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/web/client.py +1 -1
  8. yjdata-0.2.0/src/yjdata.egg-info/PKG-INFO +111 -0
  9. {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata.egg-info/SOURCES.txt +2 -1
  10. yjdata-0.2.0/tests/test_last_of_group.py +210 -0
  11. yjdata-0.1.5/PKG-INFO +0 -10
  12. yjdata-0.1.5/README.md +0 -0
  13. yjdata-0.1.5/src/yjdata.egg-info/PKG-INFO +0 -10
  14. {yjdata-0.1.5 → yjdata-0.2.0}/setup.cfg +0 -0
  15. {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/__init__.py +0 -0
  16. {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/_optional.py +0 -0
  17. {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/dataset/__init__.py +0 -0
  18. {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/dataset/quote.py +0 -0
  19. {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/dataset/snapshot.py +0 -0
  20. {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/dataset/trade.py +0 -0
  21. {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/db/__init__.py +0 -0
  22. {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/db/expression.py +0 -0
  23. {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/db/mysql.py +0 -0
  24. {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/db/postgres.py +0 -0
  25. {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/dtype.py +0 -0
  26. {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/utlity/__init__.py +0 -0
  27. {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/utlity/s3.py +0 -0
  28. {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/web/__init__.py +0 -0
  29. {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/web/_context.py +0 -0
  30. {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/web/_input.py +0 -0
  31. {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/web/calendar.py +0 -0
  32. {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/web/time_util.py +0 -0
  33. {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata.egg-info/dependency_links.txt +0 -0
  34. {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata.egg-info/requires.txt +0 -0
  35. {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata.egg-info/top_level.txt +0 -0
yjdata-0.2.0/PKG-INFO ADDED
@@ -0,0 +1,111 @@
1
+ Metadata-Version: 2.4
2
+ Name: yjdata
3
+ Version: 0.2.0
4
+ Summary: yjdata SDK for Python
5
+ Requires-Python: >=3.10
6
+ Description-Content-Type: text/markdown
7
+ Requires-Dist: httpx>=0.25
8
+ Requires-Dist: polars>=1.30
9
+ Requires-Dist: pyarrow>=18
10
+ Requires-Dist: pydantic>=2.11
11
+
12
+ # yjdata
13
+
14
+ Python SDK for yjdata, including MySQL and PostgreSQL database adapters.
15
+
16
+ ## Select the last row of each group
17
+
18
+ `MysqlAdapter` (MySQL 8.0+) and `PostgresAdapter` provide:
19
+
20
+ - `select_last_of_group(...) -> list[dict[str, Any]]`
21
+ - `select_last_of_group_polars(...) -> polars.DataFrame`
22
+
23
+ Both methods execute grouping, row selection, filtering, and projection in SQL
24
+ using `ROW_NUMBER()`. They accept the table name followed by keyword arguments:
25
+
26
+ | Parameter | Meaning |
27
+ | --- | --- |
28
+ | `table` | Table name, optionally qualified by schema or database. |
29
+ | `group_by: Sequence[str]` | Required nonempty sequence of grouping columns. |
30
+ | `sort_by: Sequence[str]` | Required nonempty sequence of sort columns, in priority order. |
31
+ | `ascending: bool \| Sequence[bool] = True` | One direction for every sort column, or one boolean per column. |
32
+ | `columns: Sequence[str] \| None = None` | Nonempty sequence of output columns; `None` returns all original columns. |
33
+ | `filter: Expression \| None = None` | Filter source rows before choosing a row per group. |
34
+ | `post_filter: Expression \| None = None` | Filter the chosen rows, before projecting output columns. |
35
+
36
+ The operation first applies `filter`, orders each group, selects its last complete
37
+ source row, applies `post_filter`, and finally projects `columns`. Values come
38
+ from the same row, including NULL values; this is not a separate last-non-NULL
39
+ aggregation for each column. A post-filter rejection removes the group entirely,
40
+ without falling back to an older row.
41
+
42
+ NULLs sort below all non-NULL values on both backends: first for ascending order
43
+ and last for descending order. Consequently, ascending selection prefers the
44
+ largest non-NULL value, while descending selection prefers NULL when present.
45
+ This rule applies to each sort column. NULL grouping keys group together.
46
+ For deterministic selection among ties, include a unique column in `sort_by`,
47
+ such as `sort_by=["change_time", "record_id"]`. Output order across groups is
48
+ unspecified.
49
+
50
+ Group, sort, and filter columns need not appear in `columns`. Internal ranking
51
+ columns are excluded by the SQL projection, even if the table has similarly
52
+ named columns. With `columns=None`, an additional zero-row query reads column
53
+ metadata so the adapter can explicitly project all original columns.
54
+ An empty result is `[]` for dictionaries or a DataFrame with no columns for
55
+ Polars, matching the existing selection methods. Polars infers types from all
56
+ returned rows.
57
+
58
+ ### THS concept snapshots
59
+
60
+ For an existing adapter `client` and a cutoff value `cutoff`:
61
+
62
+ ```python
63
+ import yjdata.db.expression as ye
64
+
65
+ data = client.select_last_of_group_polars(
66
+ table="thsetl.concept_change_record",
67
+ group_by=["concept_id"],
68
+ sort_by=["change_time"],
69
+ ascending=True,
70
+ columns=["concept_id", "concept_name"],
71
+ filter=(
72
+ (ye.col("change_time") <= cutoff)
73
+ & (ye.col("isvalid") == "1")
74
+ ),
75
+ post_filter=ye.col("change_type") != "conceptNegation",
76
+ )
77
+ ```
78
+
79
+ This selects the latest valid change at or before the cutoff, then excludes
80
+ concepts whose selected change is a negation. The MySQL query has this form:
81
+
82
+ ```sql
83
+ SELECT `concept_id`, `concept_name`
84
+ FROM (
85
+ SELECT `concept_id`, `concept_name`, `change_type`,
86
+ ROW_NUMBER() OVER (
87
+ PARTITION BY `concept_id`
88
+ ORDER BY (`change_time` IS NULL) ASC, `change_time` DESC
89
+ ) AS `_yjdata_row_number`
90
+ FROM `thsetl`.`concept_change_record`
91
+ WHERE (`change_time` <= %s) AND (`isvalid` = %s)
92
+ ) AS `_yjdata_latest`
93
+ WHERE `_yjdata_row_number` = 1 AND (`change_type` <> %s)
94
+ ```
95
+
96
+ Values are bound in order: `[cutoff, "1", "conceptNegation"]`. PostgreSQL uses
97
+ the same query structure with double-quoted identifiers. Expressions retain
98
+ normal SQL NULL semantics: for example, `change_type <> 'conceptNegation'`
99
+ also excludes NULL `change_type` values.
100
+
101
+ ## Tests
102
+
103
+ ```sh
104
+ python -m unittest discover -s tests -v
105
+ ```
106
+
107
+ The grouped-selection tests exercise both adapters' validation, quoting,
108
+ parameter ordering, and result conversion. Generated queries execute against
109
+ SQLite's window engine with DB-API placeholders translated, so tests require
110
+ neither optional database drivers nor running servers. This does not replace
111
+ integration testing against MySQL and PostgreSQL.
yjdata-0.2.0/README.md ADDED
@@ -0,0 +1,100 @@
1
+ # yjdata
2
+
3
+ Python SDK for yjdata, including MySQL and PostgreSQL database adapters.
4
+
5
+ ## Select the last row of each group
6
+
7
+ `MysqlAdapter` (MySQL 8.0+) and `PostgresAdapter` provide:
8
+
9
+ - `select_last_of_group(...) -> list[dict[str, Any]]`
10
+ - `select_last_of_group_polars(...) -> polars.DataFrame`
11
+
12
+ Both methods execute grouping, row selection, filtering, and projection in SQL
13
+ using `ROW_NUMBER()`. They accept the table name followed by keyword arguments:
14
+
15
+ | Parameter | Meaning |
16
+ | --- | --- |
17
+ | `table` | Table name, optionally qualified by schema or database. |
18
+ | `group_by: Sequence[str]` | Required nonempty sequence of grouping columns. |
19
+ | `sort_by: Sequence[str]` | Required nonempty sequence of sort columns, in priority order. |
20
+ | `ascending: bool \| Sequence[bool] = True` | One direction for every sort column, or one boolean per column. |
21
+ | `columns: Sequence[str] \| None = None` | Nonempty sequence of output columns; `None` returns all original columns. |
22
+ | `filter: Expression \| None = None` | Filter source rows before choosing a row per group. |
23
+ | `post_filter: Expression \| None = None` | Filter the chosen rows, before projecting output columns. |
24
+
25
+ The operation first applies `filter`, orders each group, selects its last complete
26
+ source row, applies `post_filter`, and finally projects `columns`. Values come
27
+ from the same row, including NULL values; this is not a separate last-non-NULL
28
+ aggregation for each column. A post-filter rejection removes the group entirely,
29
+ without falling back to an older row.
30
+
31
+ NULLs sort below all non-NULL values on both backends: first for ascending order
32
+ and last for descending order. Consequently, ascending selection prefers the
33
+ largest non-NULL value, while descending selection prefers NULL when present.
34
+ This rule applies to each sort column. NULL grouping keys group together.
35
+ For deterministic selection among ties, include a unique column in `sort_by`,
36
+ such as `sort_by=["change_time", "record_id"]`. Output order across groups is
37
+ unspecified.
38
+
39
+ Group, sort, and filter columns need not appear in `columns`. Internal ranking
40
+ columns are excluded by the SQL projection, even if the table has similarly
41
+ named columns. With `columns=None`, an additional zero-row query reads column
42
+ metadata so the adapter can explicitly project all original columns.
43
+ An empty result is `[]` for dictionaries or a DataFrame with no columns for
44
+ Polars, matching the existing selection methods. Polars infers types from all
45
+ returned rows.
46
+
47
+ ### THS concept snapshots
48
+
49
+ For an existing adapter `client` and a cutoff value `cutoff`:
50
+
51
+ ```python
52
+ import yjdata.db.expression as ye
53
+
54
+ data = client.select_last_of_group_polars(
55
+ table="thsetl.concept_change_record",
56
+ group_by=["concept_id"],
57
+ sort_by=["change_time"],
58
+ ascending=True,
59
+ columns=["concept_id", "concept_name"],
60
+ filter=(
61
+ (ye.col("change_time") <= cutoff)
62
+ & (ye.col("isvalid") == "1")
63
+ ),
64
+ post_filter=ye.col("change_type") != "conceptNegation",
65
+ )
66
+ ```
67
+
68
+ This selects the latest valid change at or before the cutoff, then excludes
69
+ concepts whose selected change is a negation. The MySQL query has this form:
70
+
71
+ ```sql
72
+ SELECT `concept_id`, `concept_name`
73
+ FROM (
74
+ SELECT `concept_id`, `concept_name`, `change_type`,
75
+ ROW_NUMBER() OVER (
76
+ PARTITION BY `concept_id`
77
+ ORDER BY (`change_time` IS NULL) ASC, `change_time` DESC
78
+ ) AS `_yjdata_row_number`
79
+ FROM `thsetl`.`concept_change_record`
80
+ WHERE (`change_time` <= %s) AND (`isvalid` = %s)
81
+ ) AS `_yjdata_latest`
82
+ WHERE `_yjdata_row_number` = 1 AND (`change_type` <> %s)
83
+ ```
84
+
85
+ Values are bound in order: `[cutoff, "1", "conceptNegation"]`. PostgreSQL uses
86
+ the same query structure with double-quoted identifiers. Expressions retain
87
+ normal SQL NULL semantics: for example, `change_type <> 'conceptNegation'`
88
+ also excludes NULL `change_type` values.
89
+
90
+ ## Tests
91
+
92
+ ```sh
93
+ python -m unittest discover -s tests -v
94
+ ```
95
+
96
+ The grouped-selection tests exercise both adapters' validation, quoting,
97
+ parameter ordering, and result conversion. Generated queries execute against
98
+ SQLite's window engine with DB-API placeholders translated, so tests require
99
+ neither optional database drivers nor running servers. This does not replace
100
+ integration testing against MySQL and PostgreSQL.
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "yjdata"
7
- version = "0.1.5"
7
+ version = "0.2.0"
8
8
  description = "yjdata SDK for Python"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.10"
@@ -227,6 +227,8 @@ def write_chunked_parquet(
227
227
  def write_seq_parquet(
228
228
  df: pl.DataFrame,
229
229
  path: str | Path,
230
+ *,
231
+ row_group_size: int = 512 * 512
230
232
  ):
231
233
  """Write rows to a Parquet file in their existing order.
232
234
 
@@ -236,6 +238,8 @@ def write_seq_parquet(
236
238
  Rows to write without additional grouping or sorting.
237
239
  path : str or pathlib.Path
238
240
  Output file path.
241
+ row_group_size: int, default 512 * 512
242
+ Number of max rows per Parquet row group.
239
243
 
240
244
  Notes
241
245
  -----
@@ -246,4 +250,5 @@ def write_seq_parquet(
246
250
  df.write_parquet(
247
251
  path,
248
252
  metadata={"ARROW:schema": encoded_schema},
253
+ row_group_size=row_group_size,
249
254
  )
@@ -25,6 +25,18 @@ def validate_delete_args(filter: Expression | None) -> None:
25
25
  raise TypeError("filter must be an Expression or None")
26
26
 
27
27
 
28
+ def validate_column_names(name: str, names: Sequence[str]) -> None:
29
+ """Require a nonempty sequence of valid column names."""
30
+ if isinstance(names, (str, bytes)) or not isinstance(names, Sequence):
31
+ raise TypeError(f"{name} must be a sequence of column names")
32
+ if not names:
33
+ raise ValueError(f"{name} must contain at least one column")
34
+ if any(not isinstance(column, str) for column in names):
35
+ raise TypeError("Column names must be strings")
36
+ if any(not column or "\x00" in column for column in names):
37
+ raise ValueError("Column names must be nonempty and contain no NUL characters")
38
+
39
+
28
40
  def validate_select_args(
29
41
  filter: Expression | None,
30
42
  columns: Sequence[str] | None,
@@ -39,14 +51,7 @@ def validate_select_args(
39
51
  for name, names in (("columns", columns), ("order_by", order_by)):
40
52
  if names is None:
41
53
  continue
42
- if isinstance(names, (str, bytes)) or not isinstance(names, Sequence):
43
- raise TypeError(f"{name} must be a sequence of column names")
44
- if not names:
45
- raise ValueError(f"{name} must contain at least one column")
46
- if any(not isinstance(column, str) for column in names):
47
- raise TypeError("Column names must be strings")
48
- if any(not column or "\x00" in column for column in names):
49
- raise ValueError("Column names must be nonempty and contain no NUL characters")
54
+ validate_column_names(name, names)
50
55
  if descending is not None:
51
56
  if isinstance(descending, (str, bytes)) or not isinstance(descending, Sequence):
52
57
  raise TypeError("descending must be a sequence of booleans")
@@ -64,6 +69,31 @@ def validate_select_args(
64
69
  raise ValueError(f"{name} must be a nonnegative integer")
65
70
 
66
71
 
72
+ def validate_last_of_group_args(
73
+ group_by: Sequence[str],
74
+ sort_by: Sequence[str],
75
+ ascending: bool | Sequence[bool],
76
+ columns: Sequence[str] | None,
77
+ filter: Expression | None,
78
+ post_filter: Expression | None,
79
+ ) -> list[bool]:
80
+ """Validate grouped selection and normalize the sort directions."""
81
+ validate_select_args(filter, columns)
82
+ validate_column_names("group_by", group_by)
83
+ validate_column_names("sort_by", sort_by)
84
+ if post_filter is not None and not isinstance(post_filter, Expression):
85
+ raise TypeError("post_filter must be an Expression or None")
86
+ if isinstance(ascending, bool):
87
+ return [ascending] * len(sort_by)
88
+ if isinstance(ascending, (str, bytes)) or not isinstance(ascending, Sequence):
89
+ raise TypeError("ascending must be a boolean or a sequence of booleans")
90
+ if any(not isinstance(direction, bool) for direction in ascending):
91
+ raise TypeError("ascending must contain only booleans")
92
+ if len(ascending) != len(sort_by):
93
+ raise ValueError("ascending must have the same length as sort_by")
94
+ return list(ascending)
95
+
96
+
67
97
  def validate_update_args(
68
98
  keys: Sequence[str],
69
99
  df: pl.DataFrame,
@@ -234,6 +264,68 @@ class DbAdapter(ABC):
234
264
  """
235
265
  raise NotImplementedError("This adapter does not implement selection")
236
266
 
267
+ def select_last_of_group(
268
+ self,
269
+ table: str,
270
+ *,
271
+ group_by: Sequence[str],
272
+ sort_by: Sequence[str],
273
+ ascending: bool | Sequence[bool] = True,
274
+ columns: Sequence[str] | None = None,
275
+ filter: Expression | None = None,
276
+ post_filter: Expression | None = None,
277
+ ) -> list[dict[str, Any]]:
278
+ """Select the last complete row of each group using SQL.
279
+
280
+ Parameters
281
+ ----------
282
+ table : str
283
+ Table name, optionally qualified by schema or database.
284
+ group_by : sequence of str
285
+ Nonempty sequence of grouping columns. NULL group keys group together.
286
+ sort_by : sequence of str
287
+ Nonempty sequence of sorting columns in priority order. Include a
288
+ unique column to make selection deterministic when values tie.
289
+ ascending : bool or sequence of bool, default True
290
+ One direction for all sort columns, or one per column. NULL is
291
+ smaller than every non-NULL value: first ascending, last descending.
292
+ columns : sequence of str, optional
293
+ Nonempty projection applied after selection and post_filter.
294
+ None returns all original columns, with no internal helper columns.
295
+ filter : Expression, optional
296
+ Condition applied before grouping and selecting rows.
297
+ post_filter : Expression, optional
298
+ Condition applied to the selected rows. A rejected row removes its
299
+ group; selection does not fall back to an earlier row.
300
+
301
+ Returns
302
+ -------
303
+ list of dict
304
+ At most one complete source row per group, projected to columns.
305
+ Group, sort and filter columns need not be returned. Result order
306
+ across groups is unspecified; an empty result is an empty list.
307
+ """
308
+ raise NotImplementedError("This adapter does not implement grouped selection")
309
+
310
+ def select_last_of_group_polars(
311
+ self,
312
+ table: str,
313
+ *,
314
+ group_by: Sequence[str],
315
+ sort_by: Sequence[str],
316
+ ascending: bool | Sequence[bool] = True,
317
+ columns: Sequence[str] | None = None,
318
+ filter: Expression | None = None,
319
+ post_filter: Expression | None = None,
320
+ ) -> pl.DataFrame:
321
+ """Select each group's last row as a Polars DataFrame.
322
+
323
+ Arguments and SQL semantics match :meth:`select_last_of_group`.
324
+ Types are inferred from all returned rows. As with select_polars,
325
+ an empty result has no columns.
326
+ """
327
+ raise NotImplementedError("This adapter does not implement grouped selection")
328
+
237
329
  @abstractmethod
238
330
  def update_polars(
239
331
  self,
@@ -1,6 +1,7 @@
1
1
  from abc import abstractmethod
2
2
  from contextlib import AbstractContextManager, contextmanager
3
3
  from threading import RLock
4
+ from textwrap import dedent
4
5
  from typing import Any, Iterator, Sequence
5
6
 
6
7
  import polars as pl
@@ -9,6 +10,7 @@ from .adapter import (
9
10
  DbAdapter,
10
11
  validate_insert_args,
11
12
  validate_delete_args,
13
+ validate_last_of_group_args,
12
14
  validate_select_args,
13
15
  validate_update_args,
14
16
  )
@@ -114,8 +116,121 @@ class SqlAdapter(DbAdapter):
114
116
  ) -> pl.DataFrame:
115
117
  return pl.DataFrame(
116
118
  self.select_data(
117
- table, filter=filter, columns=columns, order_by=order_by,
118
- descending=descending, offset=offset, limit=limit,
119
+ table,
120
+ filter=filter,
121
+ columns=columns,
122
+ order_by=order_by,
123
+ descending=descending,
124
+ offset=offset,
125
+ limit=limit,
126
+ ),
127
+ infer_schema_length=None,
128
+ )
129
+
130
+ def select_last_of_group(
131
+ self,
132
+ table: str,
133
+ *,
134
+ group_by: Sequence[str],
135
+ sort_by: Sequence[str],
136
+ ascending: bool | Sequence[bool] = True,
137
+ columns: Sequence[str] | None = None,
138
+ filter: Expression | None = None,
139
+ post_filter: Expression | None = None,
140
+ ) -> list[dict[str, Any]]:
141
+ directions = validate_last_of_group_args(
142
+ group_by,
143
+ sort_by,
144
+ ascending,
145
+ columns,
146
+ filter,
147
+ post_filter,
148
+ )
149
+ table_sql = self._table_identifier(table)
150
+ partition = ", ".join(self._quote_identifier(column) for column in group_by)
151
+ ordering = []
152
+ for column, is_ascending in zip(sort_by, directions):
153
+ term = self._quote_identifier(column)
154
+ # Reverse the requested order to rank the last row first. Explicit
155
+ # NULL flags give MySQL and PostgreSQL the same NULL-low ordering.
156
+ direction = "DESC" if is_ascending else "ASC"
157
+ null_direction = "ASC" if is_ascending else "DESC"
158
+ ordering.extend([f"({term} IS NULL) {null_direction}", f"{term} {direction}"])
159
+ where_sql = ""
160
+ args: list[Any] = []
161
+ if filter is not None:
162
+ condition, args = filter.to_param_sql(quote_identifier=self._quote_identifier)
163
+ where_sql = f"WHERE {condition}"
164
+
165
+ post_columns: list[str] = []
166
+
167
+ def quote_post_column(column: str) -> str:
168
+ post_columns.append(column)
169
+ return self._quote_identifier(column)
170
+
171
+ post_sql = ""
172
+ if post_filter is not None:
173
+ condition, post_args = post_filter.to_param_sql(quote_identifier=quote_post_column)
174
+ post_sql = f" AND ({condition})"
175
+ args.extend(post_args)
176
+
177
+ # Validate explicit projection identifiers before opening a connection.
178
+ if columns is not None:
179
+ for column in columns:
180
+ self._quote_identifier(column)
181
+ else:
182
+ # SQL has no portable SELECT * EXCEPT helper_column. Read only
183
+ # metadata so the final projection can exclude the ranking helper.
184
+ with self._connection_scope(write=False) as connection:
185
+ with self._cursor(connection) as cursor:
186
+ cursor.execute(f"SELECT * FROM {table_sql} WHERE 1 = 0", [])
187
+ columns = [description[0] for description in cursor.description]
188
+
189
+ source_columns = list(dict.fromkeys([*columns, *post_columns]))
190
+ occupied = {column.casefold() for column in source_columns}
191
+ rank_column = "_yjdata_row_number"
192
+ while rank_column.casefold() in occupied:
193
+ rank_column += "_"
194
+ rank_sql = self._quote_identifier(rank_column)
195
+ projection = ", ".join(self._quote_identifier(column) for column in columns)
196
+ source_projection = ", ".join(self._quote_identifier(column) for column in source_columns)
197
+ ordering_sql = ", ".join(ordering)
198
+ latest_sql = self._quote_identifier("_yjdata_latest")
199
+ qry = dedent(f"""
200
+ SELECT {projection}
201
+ FROM (
202
+ SELECT {source_projection},
203
+ ROW_NUMBER() OVER (
204
+ PARTITION BY {partition}
205
+ ORDER BY {ordering_sql}
206
+ ) AS {rank_sql}
207
+ FROM {table_sql}
208
+ {where_sql}
209
+ ) AS {latest_sql}
210
+ WHERE {rank_sql} = 1{post_sql}
211
+ """).strip()
212
+ return self._fetch_data(qry, args)
213
+
214
+ def select_last_of_group_polars(
215
+ self,
216
+ table: str,
217
+ *,
218
+ group_by: Sequence[str],
219
+ sort_by: Sequence[str],
220
+ ascending: bool | Sequence[bool] = True,
221
+ columns: Sequence[str] | None = None,
222
+ filter: Expression | None = None,
223
+ post_filter: Expression | None = None,
224
+ ) -> pl.DataFrame:
225
+ return pl.DataFrame(
226
+ self.select_last_of_group(
227
+ table,
228
+ group_by=group_by,
229
+ sort_by=sort_by,
230
+ ascending=ascending,
231
+ columns=columns,
232
+ filter=filter,
233
+ post_filter=post_filter,
119
234
  ),
120
235
  infer_schema_length=None,
121
236
  )
@@ -159,7 +274,10 @@ class SqlAdapter(DbAdapter):
159
274
  batch_size: int = 65536,
160
275
  ) -> None:
161
276
  update_columns = validate_update_args(
162
- keys, df, batch_size, require_update_columns=False,
277
+ keys,
278
+ df,
279
+ batch_size,
280
+ require_update_columns=False,
163
281
  )
164
282
  if df.is_empty():
165
283
  return
@@ -237,7 +355,7 @@ class SqlAdapter(DbAdapter):
237
355
  ) -> AbstractContextManager[Any]:
238
356
  pass
239
357
 
240
- def _fetch_data(self, qry: Any, args: QueryArgs = None) -> list[dict]:
358
+ def _fetch_data(self, qry: Any, args: Sequence[Any] | None = None) -> list[dict]:
241
359
  with self._connection_scope(write=False) as connection:
242
360
  with self._cursor(connection, as_dict=True) as cursor:
243
361
  cursor.execute(qry, args)
@@ -4,7 +4,7 @@ from .calendar import SyncCalenderWebClient, AsyncCalendarWebClient
4
4
  from ._context import RequestContext, SyncRequestContext, AsyncRequestContext
5
5
 
6
6
 
7
- _YJDATA_WEB_BASE_URL = 'yjdata-api.yjh-dp.com'
7
+ _YJDATA_WEB_BASE_URL = 'yjdata-api.yaojingriver.internal'
8
8
  _YJDATA_WEB_BASE_URL = os.getenv('YJDATA_WEB_BASE_URL', _YJDATA_WEB_BASE_URL)
9
9
 
10
10
 
@@ -0,0 +1,111 @@
1
+ Metadata-Version: 2.4
2
+ Name: yjdata
3
+ Version: 0.2.0
4
+ Summary: yjdata SDK for Python
5
+ Requires-Python: >=3.10
6
+ Description-Content-Type: text/markdown
7
+ Requires-Dist: httpx>=0.25
8
+ Requires-Dist: polars>=1.30
9
+ Requires-Dist: pyarrow>=18
10
+ Requires-Dist: pydantic>=2.11
11
+
12
+ # yjdata
13
+
14
+ Python SDK for yjdata, including MySQL and PostgreSQL database adapters.
15
+
16
+ ## Select the last row of each group
17
+
18
+ `MysqlAdapter` (MySQL 8.0+) and `PostgresAdapter` provide:
19
+
20
+ - `select_last_of_group(...) -> list[dict[str, Any]]`
21
+ - `select_last_of_group_polars(...) -> polars.DataFrame`
22
+
23
+ Both methods execute grouping, row selection, filtering, and projection in SQL
24
+ using `ROW_NUMBER()`. They accept the table name followed by keyword arguments:
25
+
26
+ | Parameter | Meaning |
27
+ | --- | --- |
28
+ | `table` | Table name, optionally qualified by schema or database. |
29
+ | `group_by: Sequence[str]` | Required nonempty sequence of grouping columns. |
30
+ | `sort_by: Sequence[str]` | Required nonempty sequence of sort columns, in priority order. |
31
+ | `ascending: bool \| Sequence[bool] = True` | One direction for every sort column, or one boolean per column. |
32
+ | `columns: Sequence[str] \| None = None` | Nonempty sequence of output columns; `None` returns all original columns. |
33
+ | `filter: Expression \| None = None` | Filter source rows before choosing a row per group. |
34
+ | `post_filter: Expression \| None = None` | Filter the chosen rows, before projecting output columns. |
35
+
36
+ The operation first applies `filter`, orders each group, selects its last complete
37
+ source row, applies `post_filter`, and finally projects `columns`. Values come
38
+ from the same row, including NULL values; this is not a separate last-non-NULL
39
+ aggregation for each column. A post-filter rejection removes the group entirely,
40
+ without falling back to an older row.
41
+
42
+ NULLs sort below all non-NULL values on both backends: first for ascending order
43
+ and last for descending order. Consequently, ascending selection prefers the
44
+ largest non-NULL value, while descending selection prefers NULL when present.
45
+ This rule applies to each sort column. NULL grouping keys group together.
46
+ For deterministic selection among ties, include a unique column in `sort_by`,
47
+ such as `sort_by=["change_time", "record_id"]`. Output order across groups is
48
+ unspecified.
49
+
50
+ Group, sort, and filter columns need not appear in `columns`. Internal ranking
51
+ columns are excluded by the SQL projection, even if the table has similarly
52
+ named columns. With `columns=None`, an additional zero-row query reads column
53
+ metadata so the adapter can explicitly project all original columns.
54
+ An empty result is `[]` for dictionaries or a DataFrame with no columns for
55
+ Polars, matching the existing selection methods. Polars infers types from all
56
+ returned rows.
57
+
58
+ ### THS concept snapshots
59
+
60
+ For an existing adapter `client` and a cutoff value `cutoff`:
61
+
62
+ ```python
63
+ import yjdata.db.expression as ye
64
+
65
+ data = client.select_last_of_group_polars(
66
+ table="thsetl.concept_change_record",
67
+ group_by=["concept_id"],
68
+ sort_by=["change_time"],
69
+ ascending=True,
70
+ columns=["concept_id", "concept_name"],
71
+ filter=(
72
+ (ye.col("change_time") <= cutoff)
73
+ & (ye.col("isvalid") == "1")
74
+ ),
75
+ post_filter=ye.col("change_type") != "conceptNegation",
76
+ )
77
+ ```
78
+
79
+ This selects the latest valid change at or before the cutoff, then excludes
80
+ concepts whose selected change is a negation. The MySQL query has this form:
81
+
82
+ ```sql
83
+ SELECT `concept_id`, `concept_name`
84
+ FROM (
85
+ SELECT `concept_id`, `concept_name`, `change_type`,
86
+ ROW_NUMBER() OVER (
87
+ PARTITION BY `concept_id`
88
+ ORDER BY (`change_time` IS NULL) ASC, `change_time` DESC
89
+ ) AS `_yjdata_row_number`
90
+ FROM `thsetl`.`concept_change_record`
91
+ WHERE (`change_time` <= %s) AND (`isvalid` = %s)
92
+ ) AS `_yjdata_latest`
93
+ WHERE `_yjdata_row_number` = 1 AND (`change_type` <> %s)
94
+ ```
95
+
96
+ Values are bound in order: `[cutoff, "1", "conceptNegation"]`. PostgreSQL uses
97
+ the same query structure with double-quoted identifiers. Expressions retain
98
+ normal SQL NULL semantics: for example, `change_type <> 'conceptNegation'`
99
+ also excludes NULL `change_type` values.
100
+
101
+ ## Tests
102
+
103
+ ```sh
104
+ python -m unittest discover -s tests -v
105
+ ```
106
+
107
+ The grouped-selection tests exercise both adapters' validation, quoting,
108
+ parameter ordering, and result conversion. Generated queries execute against
109
+ SQLite's window engine with DB-API placeholders translated, so tests require
110
+ neither optional database drivers nor running servers. This does not replace
111
+ integration testing against MySQL and PostgreSQL.
@@ -26,4 +26,5 @@ src/yjdata/web/_context.py
26
26
  src/yjdata/web/_input.py
27
27
  src/yjdata/web/calendar.py
28
28
  src/yjdata/web/client.py
29
- src/yjdata/web/time_util.py
29
+ src/yjdata/web/time_util.py
30
+ tests/test_last_of_group.py
@@ -0,0 +1,210 @@
1
+ """Execute both adapters' generated SQL with SQLite's window engine.
2
+
3
+ Only DB-API connection/cursor plumbing and placeholder syntax are substituted;
4
+ validation, quoting, query construction and result conversion use production code.
5
+ These tests do not require optional database drivers or running database servers.
6
+ """
7
+
8
+ import sqlite3
9
+ import unittest
10
+ from contextlib import contextmanager
11
+ from unittest.mock import Mock
12
+
13
+ import polars as pl
14
+
15
+ from yjdata.db import MysqlAdapter, PostgresAdapter, col
16
+ from yjdata.db.sql import SqlAdapter
17
+
18
+
19
+ class Cursor:
20
+ def __init__(self, connection, statements):
21
+ self.cursor = connection.cursor()
22
+ self.statements = statements
23
+
24
+ def execute(self, sql, args=None):
25
+ self.statements.append((sql, list(args or [])))
26
+ # DB-API percent escaping is also exercised, including quoted names.
27
+ sql = sql % tuple("?" for _ in (args or []))
28
+ self.cursor.execute(sql, args or [])
29
+
30
+ @property
31
+ def description(self):
32
+ return self.cursor.description
33
+
34
+ def fetchall(self):
35
+ return self.cursor.fetchall()
36
+
37
+
38
+ @contextmanager
39
+ def database(adapter_type, definition, rows):
40
+ connection = sqlite3.connect(":memory:")
41
+ connection.row_factory = sqlite3.Row
42
+ connection.execute("ATTACH DATABASE ':memory:' AS thsetl")
43
+ connection.execute(f"CREATE TABLE thsetl.records ({definition})")
44
+ if rows:
45
+ placeholders = ", ".join("?" for _ in rows[0])
46
+ connection.executemany(f"INSERT INTO thsetl.records VALUES ({placeholders})", rows)
47
+ connection.commit()
48
+ statements = []
49
+ adapter = adapter_type.__new__(adapter_type)
50
+ SqlAdapter.__init__(adapter)
51
+ adapter._create_connection = Mock(return_value=connection)
52
+ adapter._is_connection_closed = lambda conn: False
53
+
54
+ @contextmanager
55
+ def cursor(conn, *, as_dict=False):
56
+ wrapped = Cursor(conn, statements)
57
+ try:
58
+ yield wrapped
59
+ finally:
60
+ wrapped.cursor.close()
61
+
62
+ adapter._cursor = cursor
63
+ try:
64
+ yield adapter, statements
65
+ finally:
66
+ adapter.close()
67
+ connection.close()
68
+
69
+
70
+ class LastOfGroupTests(unittest.TestCase):
71
+ adapters = (MysqlAdapter, PostgresAdapter)
72
+
73
+ def test_snapshot_post_filter_does_not_fall_back(self):
74
+ rows = [
75
+ (1, "old", 1, "add", "1"),
76
+ (1, "removed", 2, "conceptNegation", "1"),
77
+ (2, "before", 1, "add", "1"),
78
+ (2, "latest", 2, "rename", "1"),
79
+ (2, "future", 4, "conceptNegation", "1"),
80
+ (2, "invalid", 3, "conceptNegation", "0"),
81
+ (3, "only invalid", 1, "add", "0"),
82
+ ]
83
+ definition = "concept_id INT, concept_name TEXT, change_time INT, change_type TEXT, isvalid TEXT"
84
+ for backend in self.adapters:
85
+ with self.subTest(backend=backend.__name__), database(backend, definition, rows) as (client, statements):
86
+ data = client.select_last_of_group_polars(
87
+ table="thsetl.records", group_by=["concept_id"],
88
+ sort_by=["change_time"], columns=["concept_id", "concept_name"],
89
+ filter=(col("change_time") <= 3) & (col("isvalid") == "1"),
90
+ post_filter=col("change_type") != "conceptNegation",
91
+ )
92
+ self.assertEqual(data.to_dicts(), [{"concept_id": 2, "concept_name": "latest"}])
93
+ self.assertEqual(len(statements), 1)
94
+ sql, args = statements[0]
95
+ self.assertEqual(args, [3, "1", "conceptNegation"])
96
+ sql = " ".join(sql.split())
97
+ q = client._quote_identifier
98
+ self.assertTrue(sql.startswith(f'SELECT {q("concept_id")}, {q("concept_name")} FROM ('))
99
+ inner, outer = sql.split(f') AS {q("_yjdata_latest")}')
100
+ self.assertNotIn("<>", inner)
101
+ self.assertIn(f'{q("change_type")} <> %s', outer)
102
+ self.assertIn(f'({q("change_time")} IS NULL) ASC, {q("change_time")} DESC', sql)
103
+
104
+ def test_null_ordering_both_directions_and_null_group_keys(self):
105
+ rows = [(1, None, "null"), (1, 2, "two"), (1, 1, "one"),
106
+ (None, 4, "four"), (None, 5, "five"), (2, None, "all null")]
107
+ for backend in self.adapters:
108
+ with self.subTest(backend=backend.__name__), database(backend, "g INT, s INT, payload TEXT", rows) as (client, _):
109
+ for ascending, expected in ((True, {"two", "five", "all null"}), (False, {"null", "four", "all null"})):
110
+ result = client.select_last_of_group(
111
+ "thsetl.records", group_by=["g"], sort_by=["s"],
112
+ ascending=ascending, columns=["payload"],
113
+ )
114
+ self.assertEqual({row["payload"] for row in result}, expected)
115
+
116
+ def test_composite_groups_mixed_directions_and_unique_tiebreaker(self):
117
+ rows = [(1, "a", 2, 3, 1, "earlier"), (1, "a", 2, 1, 2, "tie"),
118
+ (1, "a", 2, 1, 3, None), (1, "a", 1, 0, 4, "older"),
119
+ (1, "b", 0, 0, 5, "other")]
120
+ for backend in self.adapters:
121
+ with self.subTest(backend=backend.__name__), database(backend, "g INT, region TEXT, s INT, t INT, id INT, payload TEXT", rows) as (client, _):
122
+ result = client.select_last_of_group(
123
+ "thsetl.records", group_by=["g", "region"], sort_by=["s", "t", "id"],
124
+ ascending=[True, False, True], columns=["region", "id", "payload"],
125
+ )
126
+ self.assertEqual(sorted(result, key=lambda row: row["region"]), [
127
+ {"region": "a", "id": 3, "payload": None},
128
+ {"region": "b", "id": 5, "payload": "other"},
129
+ ])
130
+
131
+ def test_all_columns_preserve_real_helper_names(self):
132
+ definition = 'g INT, s INT, "_YJDATA_ROW_NUMBER" TEXT, "_yjdata_row_number_" TEXT'
133
+ for backend in self.adapters:
134
+ with self.subTest(backend=backend.__name__), database(backend, definition, [(1, 1, "real", "also real")]) as (client, statements):
135
+ result = client.select_last_of_group("thsetl.records", group_by=["g"], sort_by=["s"])
136
+ self.assertEqual(result, [{"g": 1, "s": 1, "_YJDATA_ROW_NUMBER": "real", "_yjdata_row_number_": "also real"}])
137
+ self.assertEqual(len(statements), 2)
138
+ self.assertIn("WHERE 1 = 0", statements[0][0])
139
+ self.assertIn(client._quote_identifier("_yjdata_row_number__"), statements[1][0])
140
+
141
+ def test_post_filter_helper_collision_and_or_precedence(self):
142
+ rows = [(1, 1, 1), (1, 2, 0), (2, 1, 2)]
143
+ for backend in self.adapters:
144
+ with self.subTest(backend=backend.__name__), database(backend, "g INT, s INT, _yjdata_row_number INT", rows) as (client, _):
145
+ result = client.select_last_of_group(
146
+ "thsetl.records", group_by=["g"], sort_by=["s"], columns=["g"],
147
+ post_filter=(col("_yjdata_row_number") == 1) | (col("_yjdata_row_number") == 2),
148
+ )
149
+ self.assertEqual(result, [{"g": 2}])
150
+
151
+ def test_quoted_identifiers_and_bound_literals(self):
152
+ value = "x' OR 1=1 -- %s"
153
+ for backend in self.adapters:
154
+ with self.subTest(backend=backend.__name__), database(backend, '"group" INT, "sort%" INT, "odd.name" TEXT', [(1, 1, value)]) as (client, statements):
155
+ result = client.select_last_of_group(
156
+ "thsetl.records", group_by=["group"], sort_by=["sort%"], columns=["odd.name"],
157
+ filter=col("odd.name") == value, post_filter=col("odd.name").is_in([value]),
158
+ )
159
+ self.assertEqual(result, [{"odd.name": value}])
160
+ self.assertNotIn(value, statements[0][0])
161
+ self.assertEqual(statements[0][1], [value, value])
162
+
163
+ def test_empty_results(self):
164
+ for backend in self.adapters:
165
+ for rows in ([], [(1, 1)]):
166
+ with self.subTest(backend=backend.__name__, rows=rows), database(backend, "g INT, s INT", rows) as (client, _):
167
+ kwargs = dict(group_by=["g"], sort_by=["s"], post_filter=col("g") == 99)
168
+ self.assertEqual(client.select_last_of_group("thsetl.records", **kwargs), [])
169
+ self.assertEqual(client.select_last_of_group_polars("thsetl.records", **kwargs).shape, (0, 0))
170
+
171
+ def test_polars_infers_from_all_rows(self):
172
+ for backend in self.adapters:
173
+ client = backend.__new__(backend)
174
+ client.select_last_of_group = Mock(return_value=[{"value": None}] * 150 + [{"value": 7}])
175
+ frame = client.select_last_of_group_polars("records", group_by=["g"], sort_by=["s"])
176
+ self.assertEqual(frame.schema, {"value": pl.Int64})
177
+ self.assertEqual(frame.height, 151)
178
+
179
+ def test_invalid_arguments_fail_before_connecting(self):
180
+ cases = [
181
+ ({"group_by": []}, ValueError), ({"sort_by": []}, ValueError),
182
+ ({"group_by": "g"}, TypeError), ({"sort_by": "s"}, TypeError),
183
+ ({"group_by": None}, TypeError), ({"sort_by": {"s"}}, TypeError),
184
+ ({"group_by": [1]}, TypeError), ({"sort_by": [""]}, ValueError),
185
+ ({"group_by": ["g\x00"]}, ValueError),
186
+ ({"columns": []}, ValueError), ({"columns": "g"}, TypeError),
187
+ ({"ascending": 1}, TypeError), ({"ascending": "asc"}, TypeError),
188
+ ({"ascending": [1]}, TypeError), ({"ascending": []}, ValueError),
189
+ ({"ascending": [True, False]}, ValueError),
190
+ ({"filter": True}, TypeError), ({"post_filter": False}, TypeError),
191
+ ({"filter": col("bad\x00") == 1}, ValueError),
192
+ ({"post_filter": col("bad\x00") == 1}, ValueError),
193
+ ({"table": "bad..table"}, ValueError),
194
+ ]
195
+ for backend in self.adapters:
196
+ for method in ("select_last_of_group", "select_last_of_group_polars"):
197
+ for invalid, error in cases:
198
+ with self.subTest(backend=backend.__name__, method=method, invalid=invalid):
199
+ client = backend.__new__(backend)
200
+ SqlAdapter.__init__(client)
201
+ client._create_connection = Mock(side_effect=AssertionError("must not connect"))
202
+ kwargs = dict(table="records", group_by=["g"], sort_by=["s"])
203
+ kwargs.update(invalid)
204
+ with self.assertRaises(error):
205
+ getattr(client, method)(**kwargs)
206
+ client._create_connection.assert_not_called()
207
+
208
+
209
+ if __name__ == "__main__":
210
+ unittest.main()
yjdata-0.1.5/PKG-INFO DELETED
@@ -1,10 +0,0 @@
1
- Metadata-Version: 2.4
2
- Name: yjdata
3
- Version: 0.1.5
4
- Summary: yjdata SDK for Python
5
- Requires-Python: >=3.10
6
- Description-Content-Type: text/markdown
7
- Requires-Dist: httpx>=0.25
8
- Requires-Dist: polars>=1.30
9
- Requires-Dist: pyarrow>=18
10
- Requires-Dist: pydantic>=2.11
yjdata-0.1.5/README.md DELETED
File without changes
@@ -1,10 +0,0 @@
1
- Metadata-Version: 2.4
2
- Name: yjdata
3
- Version: 0.1.5
4
- Summary: yjdata SDK for Python
5
- Requires-Python: >=3.10
6
- Description-Content-Type: text/markdown
7
- Requires-Dist: httpx>=0.25
8
- Requires-Dist: polars>=1.30
9
- Requires-Dist: pyarrow>=18
10
- Requires-Dist: pydantic>=2.11
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes