yjdata 0.1.5__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- yjdata-0.2.0/PKG-INFO +111 -0
- yjdata-0.2.0/README.md +100 -0
- {yjdata-0.1.5 → yjdata-0.2.0}/pyproject.toml +1 -1
- {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/dataset/common.py +5 -0
- {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/db/adapter.py +100 -8
- {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/db/sql.py +122 -4
- {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/web/client.py +1 -1
- yjdata-0.2.0/src/yjdata.egg-info/PKG-INFO +111 -0
- {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata.egg-info/SOURCES.txt +2 -1
- yjdata-0.2.0/tests/test_last_of_group.py +210 -0
- yjdata-0.1.5/PKG-INFO +0 -10
- yjdata-0.1.5/README.md +0 -0
- yjdata-0.1.5/src/yjdata.egg-info/PKG-INFO +0 -10
- {yjdata-0.1.5 → yjdata-0.2.0}/setup.cfg +0 -0
- {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/__init__.py +0 -0
- {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/_optional.py +0 -0
- {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/dataset/__init__.py +0 -0
- {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/dataset/quote.py +0 -0
- {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/dataset/snapshot.py +0 -0
- {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/dataset/trade.py +0 -0
- {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/db/__init__.py +0 -0
- {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/db/expression.py +0 -0
- {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/db/mysql.py +0 -0
- {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/db/postgres.py +0 -0
- {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/dtype.py +0 -0
- {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/utlity/__init__.py +0 -0
- {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/utlity/s3.py +0 -0
- {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/web/__init__.py +0 -0
- {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/web/_context.py +0 -0
- {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/web/_input.py +0 -0
- {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/web/calendar.py +0 -0
- {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata/web/time_util.py +0 -0
- {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata.egg-info/dependency_links.txt +0 -0
- {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata.egg-info/requires.txt +0 -0
- {yjdata-0.1.5 → yjdata-0.2.0}/src/yjdata.egg-info/top_level.txt +0 -0
yjdata-0.2.0/PKG-INFO
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: yjdata
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: yjdata SDK for Python
|
|
5
|
+
Requires-Python: >=3.10
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
Requires-Dist: httpx>=0.25
|
|
8
|
+
Requires-Dist: polars>=1.30
|
|
9
|
+
Requires-Dist: pyarrow>=18
|
|
10
|
+
Requires-Dist: pydantic>=2.11
|
|
11
|
+
|
|
12
|
+
# yjdata
|
|
13
|
+
|
|
14
|
+
Python SDK for yjdata, including MySQL and PostgreSQL database adapters.
|
|
15
|
+
|
|
16
|
+
## Select the last row of each group
|
|
17
|
+
|
|
18
|
+
`MysqlAdapter` (MySQL 8.0+) and `PostgresAdapter` provide:
|
|
19
|
+
|
|
20
|
+
- `select_last_of_group(...) -> list[dict[str, Any]]`
|
|
21
|
+
- `select_last_of_group_polars(...) -> polars.DataFrame`
|
|
22
|
+
|
|
23
|
+
Both methods execute grouping, row selection, filtering, and projection in SQL
|
|
24
|
+
using `ROW_NUMBER()`. They accept the table name followed by keyword arguments:
|
|
25
|
+
|
|
26
|
+
| Parameter | Meaning |
|
|
27
|
+
| --- | --- |
|
|
28
|
+
| `table` | Table name, optionally qualified by schema or database. |
|
|
29
|
+
| `group_by: Sequence[str]` | Required nonempty sequence of grouping columns. |
|
|
30
|
+
| `sort_by: Sequence[str]` | Required nonempty sequence of sort columns, in priority order. |
|
|
31
|
+
| `ascending: bool \| Sequence[bool] = True` | One direction for every sort column, or one boolean per column. |
|
|
32
|
+
| `columns: Sequence[str] \| None = None` | Nonempty sequence of output columns; `None` returns all original columns. |
|
|
33
|
+
| `filter: Expression \| None = None` | Filter source rows before choosing a row per group. |
|
|
34
|
+
| `post_filter: Expression \| None = None` | Filter the chosen rows, before projecting output columns. |
|
|
35
|
+
|
|
36
|
+
The operation first applies `filter`, orders each group, selects its last complete
|
|
37
|
+
source row, applies `post_filter`, and finally projects `columns`. Values come
|
|
38
|
+
from the same row, including NULL values; this is not a separate last-non-NULL
|
|
39
|
+
aggregation for each column. A post-filter rejection removes the group entirely,
|
|
40
|
+
without falling back to an older row.
|
|
41
|
+
|
|
42
|
+
NULLs sort below all non-NULL values on both backends: first for ascending order
|
|
43
|
+
and last for descending order. Consequently, ascending selection prefers the
|
|
44
|
+
largest non-NULL value, while descending selection prefers NULL when present.
|
|
45
|
+
This rule applies to each sort column. NULL grouping keys group together.
|
|
46
|
+
For deterministic selection among ties, include a unique column in `sort_by`,
|
|
47
|
+
such as `sort_by=["change_time", "record_id"]`. Output order across groups is
|
|
48
|
+
unspecified.
|
|
49
|
+
|
|
50
|
+
Group, sort, and filter columns need not appear in `columns`. Internal ranking
|
|
51
|
+
columns are excluded by the SQL projection, even if the table has similarly
|
|
52
|
+
named columns. With `columns=None`, an additional zero-row query reads column
|
|
53
|
+
metadata so the adapter can explicitly project all original columns.
|
|
54
|
+
An empty result is `[]` for dictionaries or a DataFrame with no columns for
|
|
55
|
+
Polars, matching the existing selection methods. Polars infers types from all
|
|
56
|
+
returned rows.
|
|
57
|
+
|
|
58
|
+
### THS concept snapshots
|
|
59
|
+
|
|
60
|
+
For an existing adapter `client` and a cutoff value `cutoff`:
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
import yjdata.db.expression as ye
|
|
64
|
+
|
|
65
|
+
data = client.select_last_of_group_polars(
|
|
66
|
+
table="thsetl.concept_change_record",
|
|
67
|
+
group_by=["concept_id"],
|
|
68
|
+
sort_by=["change_time"],
|
|
69
|
+
ascending=True,
|
|
70
|
+
columns=["concept_id", "concept_name"],
|
|
71
|
+
filter=(
|
|
72
|
+
(ye.col("change_time") <= cutoff)
|
|
73
|
+
& (ye.col("isvalid") == "1")
|
|
74
|
+
),
|
|
75
|
+
post_filter=ye.col("change_type") != "conceptNegation",
|
|
76
|
+
)
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
This selects the latest valid change at or before the cutoff, then excludes
|
|
80
|
+
concepts whose selected change is a negation. The MySQL query has this form:
|
|
81
|
+
|
|
82
|
+
```sql
|
|
83
|
+
SELECT `concept_id`, `concept_name`
|
|
84
|
+
FROM (
|
|
85
|
+
SELECT `concept_id`, `concept_name`, `change_type`,
|
|
86
|
+
ROW_NUMBER() OVER (
|
|
87
|
+
PARTITION BY `concept_id`
|
|
88
|
+
ORDER BY (`change_time` IS NULL) ASC, `change_time` DESC
|
|
89
|
+
) AS `_yjdata_row_number`
|
|
90
|
+
FROM `thsetl`.`concept_change_record`
|
|
91
|
+
WHERE (`change_time` <= %s) AND (`isvalid` = %s)
|
|
92
|
+
) AS `_yjdata_latest`
|
|
93
|
+
WHERE `_yjdata_row_number` = 1 AND (`change_type` <> %s)
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
Values are bound in order: `[cutoff, "1", "conceptNegation"]`. PostgreSQL uses
|
|
97
|
+
the same query structure with double-quoted identifiers. Expressions retain
|
|
98
|
+
normal SQL NULL semantics: for example, `change_type <> 'conceptNegation'`
|
|
99
|
+
also excludes NULL `change_type` values.
|
|
100
|
+
|
|
101
|
+
## Tests
|
|
102
|
+
|
|
103
|
+
```sh
|
|
104
|
+
python -m unittest discover -s tests -v
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
The grouped-selection tests exercise both adapters' validation, quoting,
|
|
108
|
+
parameter ordering, and result conversion. Generated queries execute against
|
|
109
|
+
SQLite's window engine with DB-API placeholders translated, so tests require
|
|
110
|
+
neither optional database drivers nor running servers. This does not replace
|
|
111
|
+
integration testing against MySQL and PostgreSQL.
|
yjdata-0.2.0/README.md
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
# yjdata
|
|
2
|
+
|
|
3
|
+
Python SDK for yjdata, including MySQL and PostgreSQL database adapters.
|
|
4
|
+
|
|
5
|
+
## Select the last row of each group
|
|
6
|
+
|
|
7
|
+
`MysqlAdapter` (MySQL 8.0+) and `PostgresAdapter` provide:
|
|
8
|
+
|
|
9
|
+
- `select_last_of_group(...) -> list[dict[str, Any]]`
|
|
10
|
+
- `select_last_of_group_polars(...) -> polars.DataFrame`
|
|
11
|
+
|
|
12
|
+
Both methods execute grouping, row selection, filtering, and projection in SQL
|
|
13
|
+
using `ROW_NUMBER()`. They accept the table name followed by keyword arguments:
|
|
14
|
+
|
|
15
|
+
| Parameter | Meaning |
|
|
16
|
+
| --- | --- |
|
|
17
|
+
| `table` | Table name, optionally qualified by schema or database. |
|
|
18
|
+
| `group_by: Sequence[str]` | Required nonempty sequence of grouping columns. |
|
|
19
|
+
| `sort_by: Sequence[str]` | Required nonempty sequence of sort columns, in priority order. |
|
|
20
|
+
| `ascending: bool \| Sequence[bool] = True` | One direction for every sort column, or one boolean per column. |
|
|
21
|
+
| `columns: Sequence[str] \| None = None` | Nonempty sequence of output columns; `None` returns all original columns. |
|
|
22
|
+
| `filter: Expression \| None = None` | Filter source rows before choosing a row per group. |
|
|
23
|
+
| `post_filter: Expression \| None = None` | Filter the chosen rows, before projecting output columns. |
|
|
24
|
+
|
|
25
|
+
The operation first applies `filter`, orders each group, selects its last complete
|
|
26
|
+
source row, applies `post_filter`, and finally projects `columns`. Values come
|
|
27
|
+
from the same row, including NULL values; this is not a separate last-non-NULL
|
|
28
|
+
aggregation for each column. A post-filter rejection removes the group entirely,
|
|
29
|
+
without falling back to an older row.
|
|
30
|
+
|
|
31
|
+
NULLs sort below all non-NULL values on both backends: first for ascending order
|
|
32
|
+
and last for descending order. Consequently, ascending selection prefers the
|
|
33
|
+
largest non-NULL value, while descending selection prefers NULL when present.
|
|
34
|
+
This rule applies to each sort column. NULL grouping keys group together.
|
|
35
|
+
For deterministic selection among ties, include a unique column in `sort_by`,
|
|
36
|
+
such as `sort_by=["change_time", "record_id"]`. Output order across groups is
|
|
37
|
+
unspecified.
|
|
38
|
+
|
|
39
|
+
Group, sort, and filter columns need not appear in `columns`. Internal ranking
|
|
40
|
+
columns are excluded by the SQL projection, even if the table has similarly
|
|
41
|
+
named columns. With `columns=None`, an additional zero-row query reads column
|
|
42
|
+
metadata so the adapter can explicitly project all original columns.
|
|
43
|
+
An empty result is `[]` for dictionaries or a DataFrame with no columns for
|
|
44
|
+
Polars, matching the existing selection methods. Polars infers types from all
|
|
45
|
+
returned rows.
|
|
46
|
+
|
|
47
|
+
### THS concept snapshots
|
|
48
|
+
|
|
49
|
+
For an existing adapter `client` and a cutoff value `cutoff`:
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
import yjdata.db.expression as ye
|
|
53
|
+
|
|
54
|
+
data = client.select_last_of_group_polars(
|
|
55
|
+
table="thsetl.concept_change_record",
|
|
56
|
+
group_by=["concept_id"],
|
|
57
|
+
sort_by=["change_time"],
|
|
58
|
+
ascending=True,
|
|
59
|
+
columns=["concept_id", "concept_name"],
|
|
60
|
+
filter=(
|
|
61
|
+
(ye.col("change_time") <= cutoff)
|
|
62
|
+
& (ye.col("isvalid") == "1")
|
|
63
|
+
),
|
|
64
|
+
post_filter=ye.col("change_type") != "conceptNegation",
|
|
65
|
+
)
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
This selects the latest valid change at or before the cutoff, then excludes
|
|
69
|
+
concepts whose selected change is a negation. The MySQL query has this form:
|
|
70
|
+
|
|
71
|
+
```sql
|
|
72
|
+
SELECT `concept_id`, `concept_name`
|
|
73
|
+
FROM (
|
|
74
|
+
SELECT `concept_id`, `concept_name`, `change_type`,
|
|
75
|
+
ROW_NUMBER() OVER (
|
|
76
|
+
PARTITION BY `concept_id`
|
|
77
|
+
ORDER BY (`change_time` IS NULL) ASC, `change_time` DESC
|
|
78
|
+
) AS `_yjdata_row_number`
|
|
79
|
+
FROM `thsetl`.`concept_change_record`
|
|
80
|
+
WHERE (`change_time` <= %s) AND (`isvalid` = %s)
|
|
81
|
+
) AS `_yjdata_latest`
|
|
82
|
+
WHERE `_yjdata_row_number` = 1 AND (`change_type` <> %s)
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
Values are bound in order: `[cutoff, "1", "conceptNegation"]`. PostgreSQL uses
|
|
86
|
+
the same query structure with double-quoted identifiers. Expressions retain
|
|
87
|
+
normal SQL NULL semantics: for example, `change_type <> 'conceptNegation'`
|
|
88
|
+
also excludes NULL `change_type` values.
|
|
89
|
+
|
|
90
|
+
## Tests
|
|
91
|
+
|
|
92
|
+
```sh
|
|
93
|
+
python -m unittest discover -s tests -v
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
The grouped-selection tests exercise both adapters' validation, quoting,
|
|
97
|
+
parameter ordering, and result conversion. Generated queries execute against
|
|
98
|
+
SQLite's window engine with DB-API placeholders translated, so tests require
|
|
99
|
+
neither optional database drivers nor running servers. This does not replace
|
|
100
|
+
integration testing against MySQL and PostgreSQL.
|
|
@@ -227,6 +227,8 @@ def write_chunked_parquet(
|
|
|
227
227
|
def write_seq_parquet(
|
|
228
228
|
df: pl.DataFrame,
|
|
229
229
|
path: str | Path,
|
|
230
|
+
*,
|
|
231
|
+
row_group_size: int = 512 * 512
|
|
230
232
|
):
|
|
231
233
|
"""Write rows to a Parquet file in their existing order.
|
|
232
234
|
|
|
@@ -236,6 +238,8 @@ def write_seq_parquet(
|
|
|
236
238
|
Rows to write without additional grouping or sorting.
|
|
237
239
|
path : str or pathlib.Path
|
|
238
240
|
Output file path.
|
|
241
|
+
row_group_size: int, default 512 * 512
|
|
242
|
+
Number of max rows per Parquet row group.
|
|
239
243
|
|
|
240
244
|
Notes
|
|
241
245
|
-----
|
|
@@ -246,4 +250,5 @@ def write_seq_parquet(
|
|
|
246
250
|
df.write_parquet(
|
|
247
251
|
path,
|
|
248
252
|
metadata={"ARROW:schema": encoded_schema},
|
|
253
|
+
row_group_size=row_group_size,
|
|
249
254
|
)
|
|
@@ -25,6 +25,18 @@ def validate_delete_args(filter: Expression | None) -> None:
|
|
|
25
25
|
raise TypeError("filter must be an Expression or None")
|
|
26
26
|
|
|
27
27
|
|
|
28
|
+
def validate_column_names(name: str, names: Sequence[str]) -> None:
|
|
29
|
+
"""Require a nonempty sequence of valid column names."""
|
|
30
|
+
if isinstance(names, (str, bytes)) or not isinstance(names, Sequence):
|
|
31
|
+
raise TypeError(f"{name} must be a sequence of column names")
|
|
32
|
+
if not names:
|
|
33
|
+
raise ValueError(f"{name} must contain at least one column")
|
|
34
|
+
if any(not isinstance(column, str) for column in names):
|
|
35
|
+
raise TypeError("Column names must be strings")
|
|
36
|
+
if any(not column or "\x00" in column for column in names):
|
|
37
|
+
raise ValueError("Column names must be nonempty and contain no NUL characters")
|
|
38
|
+
|
|
39
|
+
|
|
28
40
|
def validate_select_args(
|
|
29
41
|
filter: Expression | None,
|
|
30
42
|
columns: Sequence[str] | None,
|
|
@@ -39,14 +51,7 @@ def validate_select_args(
|
|
|
39
51
|
for name, names in (("columns", columns), ("order_by", order_by)):
|
|
40
52
|
if names is None:
|
|
41
53
|
continue
|
|
42
|
-
|
|
43
|
-
raise TypeError(f"{name} must be a sequence of column names")
|
|
44
|
-
if not names:
|
|
45
|
-
raise ValueError(f"{name} must contain at least one column")
|
|
46
|
-
if any(not isinstance(column, str) for column in names):
|
|
47
|
-
raise TypeError("Column names must be strings")
|
|
48
|
-
if any(not column or "\x00" in column for column in names):
|
|
49
|
-
raise ValueError("Column names must be nonempty and contain no NUL characters")
|
|
54
|
+
validate_column_names(name, names)
|
|
50
55
|
if descending is not None:
|
|
51
56
|
if isinstance(descending, (str, bytes)) or not isinstance(descending, Sequence):
|
|
52
57
|
raise TypeError("descending must be a sequence of booleans")
|
|
@@ -64,6 +69,31 @@ def validate_select_args(
|
|
|
64
69
|
raise ValueError(f"{name} must be a nonnegative integer")
|
|
65
70
|
|
|
66
71
|
|
|
72
|
+
def validate_last_of_group_args(
|
|
73
|
+
group_by: Sequence[str],
|
|
74
|
+
sort_by: Sequence[str],
|
|
75
|
+
ascending: bool | Sequence[bool],
|
|
76
|
+
columns: Sequence[str] | None,
|
|
77
|
+
filter: Expression | None,
|
|
78
|
+
post_filter: Expression | None,
|
|
79
|
+
) -> list[bool]:
|
|
80
|
+
"""Validate grouped selection and normalize the sort directions."""
|
|
81
|
+
validate_select_args(filter, columns)
|
|
82
|
+
validate_column_names("group_by", group_by)
|
|
83
|
+
validate_column_names("sort_by", sort_by)
|
|
84
|
+
if post_filter is not None and not isinstance(post_filter, Expression):
|
|
85
|
+
raise TypeError("post_filter must be an Expression or None")
|
|
86
|
+
if isinstance(ascending, bool):
|
|
87
|
+
return [ascending] * len(sort_by)
|
|
88
|
+
if isinstance(ascending, (str, bytes)) or not isinstance(ascending, Sequence):
|
|
89
|
+
raise TypeError("ascending must be a boolean or a sequence of booleans")
|
|
90
|
+
if any(not isinstance(direction, bool) for direction in ascending):
|
|
91
|
+
raise TypeError("ascending must contain only booleans")
|
|
92
|
+
if len(ascending) != len(sort_by):
|
|
93
|
+
raise ValueError("ascending must have the same length as sort_by")
|
|
94
|
+
return list(ascending)
|
|
95
|
+
|
|
96
|
+
|
|
67
97
|
def validate_update_args(
|
|
68
98
|
keys: Sequence[str],
|
|
69
99
|
df: pl.DataFrame,
|
|
@@ -234,6 +264,68 @@ class DbAdapter(ABC):
|
|
|
234
264
|
"""
|
|
235
265
|
raise NotImplementedError("This adapter does not implement selection")
|
|
236
266
|
|
|
267
|
+
def select_last_of_group(
|
|
268
|
+
self,
|
|
269
|
+
table: str,
|
|
270
|
+
*,
|
|
271
|
+
group_by: Sequence[str],
|
|
272
|
+
sort_by: Sequence[str],
|
|
273
|
+
ascending: bool | Sequence[bool] = True,
|
|
274
|
+
columns: Sequence[str] | None = None,
|
|
275
|
+
filter: Expression | None = None,
|
|
276
|
+
post_filter: Expression | None = None,
|
|
277
|
+
) -> list[dict[str, Any]]:
|
|
278
|
+
"""Select the last complete row of each group using SQL.
|
|
279
|
+
|
|
280
|
+
Parameters
|
|
281
|
+
----------
|
|
282
|
+
table : str
|
|
283
|
+
Table name, optionally qualified by schema or database.
|
|
284
|
+
group_by : sequence of str
|
|
285
|
+
Nonempty sequence of grouping columns. NULL group keys group together.
|
|
286
|
+
sort_by : sequence of str
|
|
287
|
+
Nonempty sequence of sorting columns in priority order. Include a
|
|
288
|
+
unique column to make selection deterministic when values tie.
|
|
289
|
+
ascending : bool or sequence of bool, default True
|
|
290
|
+
One direction for all sort columns, or one per column. NULL is
|
|
291
|
+
smaller than every non-NULL value: first ascending, last descending.
|
|
292
|
+
columns : sequence of str, optional
|
|
293
|
+
Nonempty projection applied after selection and post_filter.
|
|
294
|
+
None returns all original columns, with no internal helper columns.
|
|
295
|
+
filter : Expression, optional
|
|
296
|
+
Condition applied before grouping and selecting rows.
|
|
297
|
+
post_filter : Expression, optional
|
|
298
|
+
Condition applied to the selected rows. A rejected row removes its
|
|
299
|
+
group; selection does not fall back to an earlier row.
|
|
300
|
+
|
|
301
|
+
Returns
|
|
302
|
+
-------
|
|
303
|
+
list of dict
|
|
304
|
+
At most one complete source row per group, projected to columns.
|
|
305
|
+
Group, sort and filter columns need not be returned. Result order
|
|
306
|
+
across groups is unspecified; an empty result is an empty list.
|
|
307
|
+
"""
|
|
308
|
+
raise NotImplementedError("This adapter does not implement grouped selection")
|
|
309
|
+
|
|
310
|
+
def select_last_of_group_polars(
|
|
311
|
+
self,
|
|
312
|
+
table: str,
|
|
313
|
+
*,
|
|
314
|
+
group_by: Sequence[str],
|
|
315
|
+
sort_by: Sequence[str],
|
|
316
|
+
ascending: bool | Sequence[bool] = True,
|
|
317
|
+
columns: Sequence[str] | None = None,
|
|
318
|
+
filter: Expression | None = None,
|
|
319
|
+
post_filter: Expression | None = None,
|
|
320
|
+
) -> pl.DataFrame:
|
|
321
|
+
"""Select each group's last row as a Polars DataFrame.
|
|
322
|
+
|
|
323
|
+
Arguments and SQL semantics match :meth:`select_last_of_group`.
|
|
324
|
+
Types are inferred from all returned rows. As with select_polars,
|
|
325
|
+
an empty result has no columns.
|
|
326
|
+
"""
|
|
327
|
+
raise NotImplementedError("This adapter does not implement grouped selection")
|
|
328
|
+
|
|
237
329
|
@abstractmethod
|
|
238
330
|
def update_polars(
|
|
239
331
|
self,
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from abc import abstractmethod
|
|
2
2
|
from contextlib import AbstractContextManager, contextmanager
|
|
3
3
|
from threading import RLock
|
|
4
|
+
from textwrap import dedent
|
|
4
5
|
from typing import Any, Iterator, Sequence
|
|
5
6
|
|
|
6
7
|
import polars as pl
|
|
@@ -9,6 +10,7 @@ from .adapter import (
|
|
|
9
10
|
DbAdapter,
|
|
10
11
|
validate_insert_args,
|
|
11
12
|
validate_delete_args,
|
|
13
|
+
validate_last_of_group_args,
|
|
12
14
|
validate_select_args,
|
|
13
15
|
validate_update_args,
|
|
14
16
|
)
|
|
@@ -114,8 +116,121 @@ class SqlAdapter(DbAdapter):
|
|
|
114
116
|
) -> pl.DataFrame:
|
|
115
117
|
return pl.DataFrame(
|
|
116
118
|
self.select_data(
|
|
117
|
-
table,
|
|
118
|
-
|
|
119
|
+
table,
|
|
120
|
+
filter=filter,
|
|
121
|
+
columns=columns,
|
|
122
|
+
order_by=order_by,
|
|
123
|
+
descending=descending,
|
|
124
|
+
offset=offset,
|
|
125
|
+
limit=limit,
|
|
126
|
+
),
|
|
127
|
+
infer_schema_length=None,
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
def select_last_of_group(
|
|
131
|
+
self,
|
|
132
|
+
table: str,
|
|
133
|
+
*,
|
|
134
|
+
group_by: Sequence[str],
|
|
135
|
+
sort_by: Sequence[str],
|
|
136
|
+
ascending: bool | Sequence[bool] = True,
|
|
137
|
+
columns: Sequence[str] | None = None,
|
|
138
|
+
filter: Expression | None = None,
|
|
139
|
+
post_filter: Expression | None = None,
|
|
140
|
+
) -> list[dict[str, Any]]:
|
|
141
|
+
directions = validate_last_of_group_args(
|
|
142
|
+
group_by,
|
|
143
|
+
sort_by,
|
|
144
|
+
ascending,
|
|
145
|
+
columns,
|
|
146
|
+
filter,
|
|
147
|
+
post_filter,
|
|
148
|
+
)
|
|
149
|
+
table_sql = self._table_identifier(table)
|
|
150
|
+
partition = ", ".join(self._quote_identifier(column) for column in group_by)
|
|
151
|
+
ordering = []
|
|
152
|
+
for column, is_ascending in zip(sort_by, directions):
|
|
153
|
+
term = self._quote_identifier(column)
|
|
154
|
+
# Reverse the requested order to rank the last row first. Explicit
|
|
155
|
+
# NULL flags give MySQL and PostgreSQL the same NULL-low ordering.
|
|
156
|
+
direction = "DESC" if is_ascending else "ASC"
|
|
157
|
+
null_direction = "ASC" if is_ascending else "DESC"
|
|
158
|
+
ordering.extend([f"({term} IS NULL) {null_direction}", f"{term} {direction}"])
|
|
159
|
+
where_sql = ""
|
|
160
|
+
args: list[Any] = []
|
|
161
|
+
if filter is not None:
|
|
162
|
+
condition, args = filter.to_param_sql(quote_identifier=self._quote_identifier)
|
|
163
|
+
where_sql = f"WHERE {condition}"
|
|
164
|
+
|
|
165
|
+
post_columns: list[str] = []
|
|
166
|
+
|
|
167
|
+
def quote_post_column(column: str) -> str:
|
|
168
|
+
post_columns.append(column)
|
|
169
|
+
return self._quote_identifier(column)
|
|
170
|
+
|
|
171
|
+
post_sql = ""
|
|
172
|
+
if post_filter is not None:
|
|
173
|
+
condition, post_args = post_filter.to_param_sql(quote_identifier=quote_post_column)
|
|
174
|
+
post_sql = f" AND ({condition})"
|
|
175
|
+
args.extend(post_args)
|
|
176
|
+
|
|
177
|
+
# Validate explicit projection identifiers before opening a connection.
|
|
178
|
+
if columns is not None:
|
|
179
|
+
for column in columns:
|
|
180
|
+
self._quote_identifier(column)
|
|
181
|
+
else:
|
|
182
|
+
# SQL has no portable SELECT * EXCEPT helper_column. Read only
|
|
183
|
+
# metadata so the final projection can exclude the ranking helper.
|
|
184
|
+
with self._connection_scope(write=False) as connection:
|
|
185
|
+
with self._cursor(connection) as cursor:
|
|
186
|
+
cursor.execute(f"SELECT * FROM {table_sql} WHERE 1 = 0", [])
|
|
187
|
+
columns = [description[0] for description in cursor.description]
|
|
188
|
+
|
|
189
|
+
source_columns = list(dict.fromkeys([*columns, *post_columns]))
|
|
190
|
+
occupied = {column.casefold() for column in source_columns}
|
|
191
|
+
rank_column = "_yjdata_row_number"
|
|
192
|
+
while rank_column.casefold() in occupied:
|
|
193
|
+
rank_column += "_"
|
|
194
|
+
rank_sql = self._quote_identifier(rank_column)
|
|
195
|
+
projection = ", ".join(self._quote_identifier(column) for column in columns)
|
|
196
|
+
source_projection = ", ".join(self._quote_identifier(column) for column in source_columns)
|
|
197
|
+
ordering_sql = ", ".join(ordering)
|
|
198
|
+
latest_sql = self._quote_identifier("_yjdata_latest")
|
|
199
|
+
qry = dedent(f"""
|
|
200
|
+
SELECT {projection}
|
|
201
|
+
FROM (
|
|
202
|
+
SELECT {source_projection},
|
|
203
|
+
ROW_NUMBER() OVER (
|
|
204
|
+
PARTITION BY {partition}
|
|
205
|
+
ORDER BY {ordering_sql}
|
|
206
|
+
) AS {rank_sql}
|
|
207
|
+
FROM {table_sql}
|
|
208
|
+
{where_sql}
|
|
209
|
+
) AS {latest_sql}
|
|
210
|
+
WHERE {rank_sql} = 1{post_sql}
|
|
211
|
+
""").strip()
|
|
212
|
+
return self._fetch_data(qry, args)
|
|
213
|
+
|
|
214
|
+
def select_last_of_group_polars(
|
|
215
|
+
self,
|
|
216
|
+
table: str,
|
|
217
|
+
*,
|
|
218
|
+
group_by: Sequence[str],
|
|
219
|
+
sort_by: Sequence[str],
|
|
220
|
+
ascending: bool | Sequence[bool] = True,
|
|
221
|
+
columns: Sequence[str] | None = None,
|
|
222
|
+
filter: Expression | None = None,
|
|
223
|
+
post_filter: Expression | None = None,
|
|
224
|
+
) -> pl.DataFrame:
|
|
225
|
+
return pl.DataFrame(
|
|
226
|
+
self.select_last_of_group(
|
|
227
|
+
table,
|
|
228
|
+
group_by=group_by,
|
|
229
|
+
sort_by=sort_by,
|
|
230
|
+
ascending=ascending,
|
|
231
|
+
columns=columns,
|
|
232
|
+
filter=filter,
|
|
233
|
+
post_filter=post_filter,
|
|
119
234
|
),
|
|
120
235
|
infer_schema_length=None,
|
|
121
236
|
)
|
|
@@ -159,7 +274,10 @@ class SqlAdapter(DbAdapter):
|
|
|
159
274
|
batch_size: int = 65536,
|
|
160
275
|
) -> None:
|
|
161
276
|
update_columns = validate_update_args(
|
|
162
|
-
keys,
|
|
277
|
+
keys,
|
|
278
|
+
df,
|
|
279
|
+
batch_size,
|
|
280
|
+
require_update_columns=False,
|
|
163
281
|
)
|
|
164
282
|
if df.is_empty():
|
|
165
283
|
return
|
|
@@ -237,7 +355,7 @@ class SqlAdapter(DbAdapter):
|
|
|
237
355
|
) -> AbstractContextManager[Any]:
|
|
238
356
|
pass
|
|
239
357
|
|
|
240
|
-
def _fetch_data(self, qry: Any, args:
|
|
358
|
+
def _fetch_data(self, qry: Any, args: Sequence[Any] | None = None) -> list[dict]:
|
|
241
359
|
with self._connection_scope(write=False) as connection:
|
|
242
360
|
with self._cursor(connection, as_dict=True) as cursor:
|
|
243
361
|
cursor.execute(qry, args)
|
|
@@ -4,7 +4,7 @@ from .calendar import SyncCalenderWebClient, AsyncCalendarWebClient
|
|
|
4
4
|
from ._context import RequestContext, SyncRequestContext, AsyncRequestContext
|
|
5
5
|
|
|
6
6
|
|
|
7
|
-
_YJDATA_WEB_BASE_URL = 'yjdata-api.
|
|
7
|
+
_YJDATA_WEB_BASE_URL = 'yjdata-api.yaojingriver.internal'
|
|
8
8
|
_YJDATA_WEB_BASE_URL = os.getenv('YJDATA_WEB_BASE_URL', _YJDATA_WEB_BASE_URL)
|
|
9
9
|
|
|
10
10
|
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: yjdata
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: yjdata SDK for Python
|
|
5
|
+
Requires-Python: >=3.10
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
Requires-Dist: httpx>=0.25
|
|
8
|
+
Requires-Dist: polars>=1.30
|
|
9
|
+
Requires-Dist: pyarrow>=18
|
|
10
|
+
Requires-Dist: pydantic>=2.11
|
|
11
|
+
|
|
12
|
+
# yjdata
|
|
13
|
+
|
|
14
|
+
Python SDK for yjdata, including MySQL and PostgreSQL database adapters.
|
|
15
|
+
|
|
16
|
+
## Select the last row of each group
|
|
17
|
+
|
|
18
|
+
`MysqlAdapter` (MySQL 8.0+) and `PostgresAdapter` provide:
|
|
19
|
+
|
|
20
|
+
- `select_last_of_group(...) -> list[dict[str, Any]]`
|
|
21
|
+
- `select_last_of_group_polars(...) -> polars.DataFrame`
|
|
22
|
+
|
|
23
|
+
Both methods execute grouping, row selection, filtering, and projection in SQL
|
|
24
|
+
using `ROW_NUMBER()`. They accept the table name followed by keyword arguments:
|
|
25
|
+
|
|
26
|
+
| Parameter | Meaning |
|
|
27
|
+
| --- | --- |
|
|
28
|
+
| `table` | Table name, optionally qualified by schema or database. |
|
|
29
|
+
| `group_by: Sequence[str]` | Required nonempty sequence of grouping columns. |
|
|
30
|
+
| `sort_by: Sequence[str]` | Required nonempty sequence of sort columns, in priority order. |
|
|
31
|
+
| `ascending: bool \| Sequence[bool] = True` | One direction for every sort column, or one boolean per column. |
|
|
32
|
+
| `columns: Sequence[str] \| None = None` | Nonempty sequence of output columns; `None` returns all original columns. |
|
|
33
|
+
| `filter: Expression \| None = None` | Filter source rows before choosing a row per group. |
|
|
34
|
+
| `post_filter: Expression \| None = None` | Filter the chosen rows, before projecting output columns. |
|
|
35
|
+
|
|
36
|
+
The operation first applies `filter`, orders each group, selects its last complete
|
|
37
|
+
source row, applies `post_filter`, and finally projects `columns`. Values come
|
|
38
|
+
from the same row, including NULL values; this is not a separate last-non-NULL
|
|
39
|
+
aggregation for each column. A post-filter rejection removes the group entirely,
|
|
40
|
+
without falling back to an older row.
|
|
41
|
+
|
|
42
|
+
NULLs sort below all non-NULL values on both backends: first for ascending order
|
|
43
|
+
and last for descending order. Consequently, ascending selection prefers the
|
|
44
|
+
largest non-NULL value, while descending selection prefers NULL when present.
|
|
45
|
+
This rule applies to each sort column. NULL grouping keys group together.
|
|
46
|
+
For deterministic selection among ties, include a unique column in `sort_by`,
|
|
47
|
+
such as `sort_by=["change_time", "record_id"]`. Output order across groups is
|
|
48
|
+
unspecified.
|
|
49
|
+
|
|
50
|
+
Group, sort, and filter columns need not appear in `columns`. Internal ranking
|
|
51
|
+
columns are excluded by the SQL projection, even if the table has similarly
|
|
52
|
+
named columns. With `columns=None`, an additional zero-row query reads column
|
|
53
|
+
metadata so the adapter can explicitly project all original columns.
|
|
54
|
+
An empty result is `[]` for dictionaries or a DataFrame with no columns for
|
|
55
|
+
Polars, matching the existing selection methods. Polars infers types from all
|
|
56
|
+
returned rows.
|
|
57
|
+
|
|
58
|
+
### THS concept snapshots
|
|
59
|
+
|
|
60
|
+
For an existing adapter `client` and a cutoff value `cutoff`:
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
import yjdata.db.expression as ye
|
|
64
|
+
|
|
65
|
+
data = client.select_last_of_group_polars(
|
|
66
|
+
table="thsetl.concept_change_record",
|
|
67
|
+
group_by=["concept_id"],
|
|
68
|
+
sort_by=["change_time"],
|
|
69
|
+
ascending=True,
|
|
70
|
+
columns=["concept_id", "concept_name"],
|
|
71
|
+
filter=(
|
|
72
|
+
(ye.col("change_time") <= cutoff)
|
|
73
|
+
& (ye.col("isvalid") == "1")
|
|
74
|
+
),
|
|
75
|
+
post_filter=ye.col("change_type") != "conceptNegation",
|
|
76
|
+
)
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
This selects the latest valid change at or before the cutoff, then excludes
|
|
80
|
+
concepts whose selected change is a negation. The MySQL query has this form:
|
|
81
|
+
|
|
82
|
+
```sql
|
|
83
|
+
SELECT `concept_id`, `concept_name`
|
|
84
|
+
FROM (
|
|
85
|
+
SELECT `concept_id`, `concept_name`, `change_type`,
|
|
86
|
+
ROW_NUMBER() OVER (
|
|
87
|
+
PARTITION BY `concept_id`
|
|
88
|
+
ORDER BY (`change_time` IS NULL) ASC, `change_time` DESC
|
|
89
|
+
) AS `_yjdata_row_number`
|
|
90
|
+
FROM `thsetl`.`concept_change_record`
|
|
91
|
+
WHERE (`change_time` <= %s) AND (`isvalid` = %s)
|
|
92
|
+
) AS `_yjdata_latest`
|
|
93
|
+
WHERE `_yjdata_row_number` = 1 AND (`change_type` <> %s)
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
Values are bound in order: `[cutoff, "1", "conceptNegation"]`. PostgreSQL uses
|
|
97
|
+
the same query structure with double-quoted identifiers. Expressions retain
|
|
98
|
+
normal SQL NULL semantics: for example, `change_type <> 'conceptNegation'`
|
|
99
|
+
also excludes NULL `change_type` values.
|
|
100
|
+
|
|
101
|
+
## Tests
|
|
102
|
+
|
|
103
|
+
```sh
|
|
104
|
+
python -m unittest discover -s tests -v
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
The grouped-selection tests exercise both adapters' validation, quoting,
|
|
108
|
+
parameter ordering, and result conversion. Generated queries execute against
|
|
109
|
+
SQLite's window engine with DB-API placeholders translated, so tests require
|
|
110
|
+
neither optional database drivers nor running servers. This does not replace
|
|
111
|
+
integration testing against MySQL and PostgreSQL.
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
"""Execute both adapters' generated SQL with SQLite's window engine.
|
|
2
|
+
|
|
3
|
+
Only DB-API connection/cursor plumbing and placeholder syntax are substituted;
|
|
4
|
+
validation, quoting, query construction and result conversion use production code.
|
|
5
|
+
These tests do not require optional database drivers or running database servers.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import sqlite3
|
|
9
|
+
import unittest
|
|
10
|
+
from contextlib import contextmanager
|
|
11
|
+
from unittest.mock import Mock
|
|
12
|
+
|
|
13
|
+
import polars as pl
|
|
14
|
+
|
|
15
|
+
from yjdata.db import MysqlAdapter, PostgresAdapter, col
|
|
16
|
+
from yjdata.db.sql import SqlAdapter
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class Cursor:
|
|
20
|
+
def __init__(self, connection, statements):
|
|
21
|
+
self.cursor = connection.cursor()
|
|
22
|
+
self.statements = statements
|
|
23
|
+
|
|
24
|
+
def execute(self, sql, args=None):
|
|
25
|
+
self.statements.append((sql, list(args or [])))
|
|
26
|
+
# DB-API percent escaping is also exercised, including quoted names.
|
|
27
|
+
sql = sql % tuple("?" for _ in (args or []))
|
|
28
|
+
self.cursor.execute(sql, args or [])
|
|
29
|
+
|
|
30
|
+
@property
|
|
31
|
+
def description(self):
|
|
32
|
+
return self.cursor.description
|
|
33
|
+
|
|
34
|
+
def fetchall(self):
|
|
35
|
+
return self.cursor.fetchall()
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@contextmanager
|
|
39
|
+
def database(adapter_type, definition, rows):
|
|
40
|
+
connection = sqlite3.connect(":memory:")
|
|
41
|
+
connection.row_factory = sqlite3.Row
|
|
42
|
+
connection.execute("ATTACH DATABASE ':memory:' AS thsetl")
|
|
43
|
+
connection.execute(f"CREATE TABLE thsetl.records ({definition})")
|
|
44
|
+
if rows:
|
|
45
|
+
placeholders = ", ".join("?" for _ in rows[0])
|
|
46
|
+
connection.executemany(f"INSERT INTO thsetl.records VALUES ({placeholders})", rows)
|
|
47
|
+
connection.commit()
|
|
48
|
+
statements = []
|
|
49
|
+
adapter = adapter_type.__new__(adapter_type)
|
|
50
|
+
SqlAdapter.__init__(adapter)
|
|
51
|
+
adapter._create_connection = Mock(return_value=connection)
|
|
52
|
+
adapter._is_connection_closed = lambda conn: False
|
|
53
|
+
|
|
54
|
+
@contextmanager
|
|
55
|
+
def cursor(conn, *, as_dict=False):
|
|
56
|
+
wrapped = Cursor(conn, statements)
|
|
57
|
+
try:
|
|
58
|
+
yield wrapped
|
|
59
|
+
finally:
|
|
60
|
+
wrapped.cursor.close()
|
|
61
|
+
|
|
62
|
+
adapter._cursor = cursor
|
|
63
|
+
try:
|
|
64
|
+
yield adapter, statements
|
|
65
|
+
finally:
|
|
66
|
+
adapter.close()
|
|
67
|
+
connection.close()
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class LastOfGroupTests(unittest.TestCase):
|
|
71
|
+
adapters = (MysqlAdapter, PostgresAdapter)
|
|
72
|
+
|
|
73
|
+
def test_snapshot_post_filter_does_not_fall_back(self):
|
|
74
|
+
rows = [
|
|
75
|
+
(1, "old", 1, "add", "1"),
|
|
76
|
+
(1, "removed", 2, "conceptNegation", "1"),
|
|
77
|
+
(2, "before", 1, "add", "1"),
|
|
78
|
+
(2, "latest", 2, "rename", "1"),
|
|
79
|
+
(2, "future", 4, "conceptNegation", "1"),
|
|
80
|
+
(2, "invalid", 3, "conceptNegation", "0"),
|
|
81
|
+
(3, "only invalid", 1, "add", "0"),
|
|
82
|
+
]
|
|
83
|
+
definition = "concept_id INT, concept_name TEXT, change_time INT, change_type TEXT, isvalid TEXT"
|
|
84
|
+
for backend in self.adapters:
|
|
85
|
+
with self.subTest(backend=backend.__name__), database(backend, definition, rows) as (client, statements):
|
|
86
|
+
data = client.select_last_of_group_polars(
|
|
87
|
+
table="thsetl.records", group_by=["concept_id"],
|
|
88
|
+
sort_by=["change_time"], columns=["concept_id", "concept_name"],
|
|
89
|
+
filter=(col("change_time") <= 3) & (col("isvalid") == "1"),
|
|
90
|
+
post_filter=col("change_type") != "conceptNegation",
|
|
91
|
+
)
|
|
92
|
+
self.assertEqual(data.to_dicts(), [{"concept_id": 2, "concept_name": "latest"}])
|
|
93
|
+
self.assertEqual(len(statements), 1)
|
|
94
|
+
sql, args = statements[0]
|
|
95
|
+
self.assertEqual(args, [3, "1", "conceptNegation"])
|
|
96
|
+
sql = " ".join(sql.split())
|
|
97
|
+
q = client._quote_identifier
|
|
98
|
+
self.assertTrue(sql.startswith(f'SELECT {q("concept_id")}, {q("concept_name")} FROM ('))
|
|
99
|
+
inner, outer = sql.split(f') AS {q("_yjdata_latest")}')
|
|
100
|
+
self.assertNotIn("<>", inner)
|
|
101
|
+
self.assertIn(f'{q("change_type")} <> %s', outer)
|
|
102
|
+
self.assertIn(f'({q("change_time")} IS NULL) ASC, {q("change_time")} DESC', sql)
|
|
103
|
+
|
|
104
|
+
def test_null_ordering_both_directions_and_null_group_keys(self):
|
|
105
|
+
rows = [(1, None, "null"), (1, 2, "two"), (1, 1, "one"),
|
|
106
|
+
(None, 4, "four"), (None, 5, "five"), (2, None, "all null")]
|
|
107
|
+
for backend in self.adapters:
|
|
108
|
+
with self.subTest(backend=backend.__name__), database(backend, "g INT, s INT, payload TEXT", rows) as (client, _):
|
|
109
|
+
for ascending, expected in ((True, {"two", "five", "all null"}), (False, {"null", "four", "all null"})):
|
|
110
|
+
result = client.select_last_of_group(
|
|
111
|
+
"thsetl.records", group_by=["g"], sort_by=["s"],
|
|
112
|
+
ascending=ascending, columns=["payload"],
|
|
113
|
+
)
|
|
114
|
+
self.assertEqual({row["payload"] for row in result}, expected)
|
|
115
|
+
|
|
116
|
+
def test_composite_groups_mixed_directions_and_unique_tiebreaker(self):
|
|
117
|
+
rows = [(1, "a", 2, 3, 1, "earlier"), (1, "a", 2, 1, 2, "tie"),
|
|
118
|
+
(1, "a", 2, 1, 3, None), (1, "a", 1, 0, 4, "older"),
|
|
119
|
+
(1, "b", 0, 0, 5, "other")]
|
|
120
|
+
for backend in self.adapters:
|
|
121
|
+
with self.subTest(backend=backend.__name__), database(backend, "g INT, region TEXT, s INT, t INT, id INT, payload TEXT", rows) as (client, _):
|
|
122
|
+
result = client.select_last_of_group(
|
|
123
|
+
"thsetl.records", group_by=["g", "region"], sort_by=["s", "t", "id"],
|
|
124
|
+
ascending=[True, False, True], columns=["region", "id", "payload"],
|
|
125
|
+
)
|
|
126
|
+
self.assertEqual(sorted(result, key=lambda row: row["region"]), [
|
|
127
|
+
{"region": "a", "id": 3, "payload": None},
|
|
128
|
+
{"region": "b", "id": 5, "payload": "other"},
|
|
129
|
+
])
|
|
130
|
+
|
|
131
|
+
def test_all_columns_preserve_real_helper_names(self):
|
|
132
|
+
definition = 'g INT, s INT, "_YJDATA_ROW_NUMBER" TEXT, "_yjdata_row_number_" TEXT'
|
|
133
|
+
for backend in self.adapters:
|
|
134
|
+
with self.subTest(backend=backend.__name__), database(backend, definition, [(1, 1, "real", "also real")]) as (client, statements):
|
|
135
|
+
result = client.select_last_of_group("thsetl.records", group_by=["g"], sort_by=["s"])
|
|
136
|
+
self.assertEqual(result, [{"g": 1, "s": 1, "_YJDATA_ROW_NUMBER": "real", "_yjdata_row_number_": "also real"}])
|
|
137
|
+
self.assertEqual(len(statements), 2)
|
|
138
|
+
self.assertIn("WHERE 1 = 0", statements[0][0])
|
|
139
|
+
self.assertIn(client._quote_identifier("_yjdata_row_number__"), statements[1][0])
|
|
140
|
+
|
|
141
|
+
def test_post_filter_helper_collision_and_or_precedence(self):
|
|
142
|
+
rows = [(1, 1, 1), (1, 2, 0), (2, 1, 2)]
|
|
143
|
+
for backend in self.adapters:
|
|
144
|
+
with self.subTest(backend=backend.__name__), database(backend, "g INT, s INT, _yjdata_row_number INT", rows) as (client, _):
|
|
145
|
+
result = client.select_last_of_group(
|
|
146
|
+
"thsetl.records", group_by=["g"], sort_by=["s"], columns=["g"],
|
|
147
|
+
post_filter=(col("_yjdata_row_number") == 1) | (col("_yjdata_row_number") == 2),
|
|
148
|
+
)
|
|
149
|
+
self.assertEqual(result, [{"g": 2}])
|
|
150
|
+
|
|
151
|
+
def test_quoted_identifiers_and_bound_literals(self):
|
|
152
|
+
value = "x' OR 1=1 -- %s"
|
|
153
|
+
for backend in self.adapters:
|
|
154
|
+
with self.subTest(backend=backend.__name__), database(backend, '"group" INT, "sort%" INT, "odd.name" TEXT', [(1, 1, value)]) as (client, statements):
|
|
155
|
+
result = client.select_last_of_group(
|
|
156
|
+
"thsetl.records", group_by=["group"], sort_by=["sort%"], columns=["odd.name"],
|
|
157
|
+
filter=col("odd.name") == value, post_filter=col("odd.name").is_in([value]),
|
|
158
|
+
)
|
|
159
|
+
self.assertEqual(result, [{"odd.name": value}])
|
|
160
|
+
self.assertNotIn(value, statements[0][0])
|
|
161
|
+
self.assertEqual(statements[0][1], [value, value])
|
|
162
|
+
|
|
163
|
+
def test_empty_results(self):
|
|
164
|
+
for backend in self.adapters:
|
|
165
|
+
for rows in ([], [(1, 1)]):
|
|
166
|
+
with self.subTest(backend=backend.__name__, rows=rows), database(backend, "g INT, s INT", rows) as (client, _):
|
|
167
|
+
kwargs = dict(group_by=["g"], sort_by=["s"], post_filter=col("g") == 99)
|
|
168
|
+
self.assertEqual(client.select_last_of_group("thsetl.records", **kwargs), [])
|
|
169
|
+
self.assertEqual(client.select_last_of_group_polars("thsetl.records", **kwargs).shape, (0, 0))
|
|
170
|
+
|
|
171
|
+
def test_polars_infers_from_all_rows(self):
|
|
172
|
+
for backend in self.adapters:
|
|
173
|
+
client = backend.__new__(backend)
|
|
174
|
+
client.select_last_of_group = Mock(return_value=[{"value": None}] * 150 + [{"value": 7}])
|
|
175
|
+
frame = client.select_last_of_group_polars("records", group_by=["g"], sort_by=["s"])
|
|
176
|
+
self.assertEqual(frame.schema, {"value": pl.Int64})
|
|
177
|
+
self.assertEqual(frame.height, 151)
|
|
178
|
+
|
|
179
|
+
def test_invalid_arguments_fail_before_connecting(self):
|
|
180
|
+
cases = [
|
|
181
|
+
({"group_by": []}, ValueError), ({"sort_by": []}, ValueError),
|
|
182
|
+
({"group_by": "g"}, TypeError), ({"sort_by": "s"}, TypeError),
|
|
183
|
+
({"group_by": None}, TypeError), ({"sort_by": {"s"}}, TypeError),
|
|
184
|
+
({"group_by": [1]}, TypeError), ({"sort_by": [""]}, ValueError),
|
|
185
|
+
({"group_by": ["g\x00"]}, ValueError),
|
|
186
|
+
({"columns": []}, ValueError), ({"columns": "g"}, TypeError),
|
|
187
|
+
({"ascending": 1}, TypeError), ({"ascending": "asc"}, TypeError),
|
|
188
|
+
({"ascending": [1]}, TypeError), ({"ascending": []}, ValueError),
|
|
189
|
+
({"ascending": [True, False]}, ValueError),
|
|
190
|
+
({"filter": True}, TypeError), ({"post_filter": False}, TypeError),
|
|
191
|
+
({"filter": col("bad\x00") == 1}, ValueError),
|
|
192
|
+
({"post_filter": col("bad\x00") == 1}, ValueError),
|
|
193
|
+
({"table": "bad..table"}, ValueError),
|
|
194
|
+
]
|
|
195
|
+
for backend in self.adapters:
|
|
196
|
+
for method in ("select_last_of_group", "select_last_of_group_polars"):
|
|
197
|
+
for invalid, error in cases:
|
|
198
|
+
with self.subTest(backend=backend.__name__, method=method, invalid=invalid):
|
|
199
|
+
client = backend.__new__(backend)
|
|
200
|
+
SqlAdapter.__init__(client)
|
|
201
|
+
client._create_connection = Mock(side_effect=AssertionError("must not connect"))
|
|
202
|
+
kwargs = dict(table="records", group_by=["g"], sort_by=["s"])
|
|
203
|
+
kwargs.update(invalid)
|
|
204
|
+
with self.assertRaises(error):
|
|
205
|
+
getattr(client, method)(**kwargs)
|
|
206
|
+
client._create_connection.assert_not_called()
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
if __name__ == "__main__":
|
|
210
|
+
unittest.main()
|
yjdata-0.1.5/PKG-INFO
DELETED
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: yjdata
|
|
3
|
-
Version: 0.1.5
|
|
4
|
-
Summary: yjdata SDK for Python
|
|
5
|
-
Requires-Python: >=3.10
|
|
6
|
-
Description-Content-Type: text/markdown
|
|
7
|
-
Requires-Dist: httpx>=0.25
|
|
8
|
-
Requires-Dist: polars>=1.30
|
|
9
|
-
Requires-Dist: pyarrow>=18
|
|
10
|
-
Requires-Dist: pydantic>=2.11
|
yjdata-0.1.5/README.md
DELETED
|
File without changes
|
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: yjdata
|
|
3
|
-
Version: 0.1.5
|
|
4
|
-
Summary: yjdata SDK for Python
|
|
5
|
-
Requires-Python: >=3.10
|
|
6
|
-
Description-Content-Type: text/markdown
|
|
7
|
-
Requires-Dist: httpx>=0.25
|
|
8
|
-
Requires-Dist: polars>=1.30
|
|
9
|
-
Requires-Dist: pyarrow>=18
|
|
10
|
-
Requires-Dist: pydantic>=2.11
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|