thds.core 1.38.20250617225213__py3-none-any.whl → 1.38.20250618170443__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of thds.core might be problematic. Click here for more details.
- thds/core/inspect.py +2 -3
- thds/core/meta.py +14 -153
- thds/core/sqlite/insert_utils.py +1 -1
- {thds_core-1.38.20250617225213.dist-info → thds_core-1.38.20250618170443.dist-info}/METADATA +1 -1
- {thds_core-1.38.20250617225213.dist-info → thds_core-1.38.20250618170443.dist-info}/RECORD +8 -8
- {thds_core-1.38.20250617225213.dist-info → thds_core-1.38.20250618170443.dist-info}/WHEEL +0 -0
- {thds_core-1.38.20250617225213.dist-info → thds_core-1.38.20250618170443.dist-info}/entry_points.txt +0 -0
- {thds_core-1.38.20250617225213.dist-info → thds_core-1.38.20250618170443.dist-info}/top_level.txt +0 -0
thds/core/inspect.py
CHANGED
thds/core/meta.py
CHANGED
|
@@ -1,22 +1,26 @@
|
|
|
1
|
+
"""Some parts of this module having to do with actual metadata files have been removed,
|
|
2
|
+
being no longer in use.
|
|
3
|
+
|
|
4
|
+
The actual removal was done mainly to enable us to remove attrs and cattrs as dependencies,
|
|
5
|
+
not because we don't like them, but because reducing our depedency footprint in
|
|
6
|
+
our most central library is a good idea.
|
|
7
|
+
"""
|
|
8
|
+
|
|
1
9
|
import importlib
|
|
2
|
-
import json
|
|
3
10
|
import os
|
|
4
11
|
import re
|
|
5
12
|
import typing as ty
|
|
13
|
+
from dataclasses import dataclass, field
|
|
6
14
|
from datetime import datetime, timezone
|
|
7
15
|
from functools import lru_cache
|
|
8
16
|
from getpass import getuser
|
|
9
17
|
from importlib.metadata import PackageNotFoundError, version
|
|
10
|
-
from importlib.resources import Package
|
|
18
|
+
from importlib.resources import Package
|
|
11
19
|
from pathlib import Path
|
|
12
20
|
from types import MappingProxyType
|
|
13
21
|
|
|
14
|
-
import attrs
|
|
15
|
-
from cattrs import Converter
|
|
16
|
-
|
|
17
22
|
from . import calgitver, git
|
|
18
23
|
from .log import getLogger
|
|
19
|
-
from .types import StrOrPath
|
|
20
24
|
|
|
21
25
|
LayoutType = ty.Literal["flat", "src"]
|
|
22
26
|
NameFormatType = ty.Literal["git", "docker", "hive"]
|
|
@@ -37,8 +41,6 @@ GIT_IS_DIRTY = "GIT_IS_DIRTY"
|
|
|
37
41
|
GIT_BRANCH = "GIT_BRANCH"
|
|
38
42
|
THDS_USER = "THDS_USER"
|
|
39
43
|
|
|
40
|
-
META_FILE = "meta.json"
|
|
41
|
-
|
|
42
44
|
LOGGER = getLogger(__name__)
|
|
43
45
|
|
|
44
46
|
|
|
@@ -170,12 +172,6 @@ def get_version(pkg: Package, orig: str = "") -> str:
|
|
|
170
172
|
# 'recurse' upward, assuming that the package name is overly-specified
|
|
171
173
|
pkg_ = pkg.split(".")
|
|
172
174
|
if len(pkg_) <= 1:
|
|
173
|
-
# Check to see if there's a
|
|
174
|
-
# meta.json file hanging around, and if so, see if it contains a pyproject_version.
|
|
175
|
-
metadata = read_metadata(orig or pkg)
|
|
176
|
-
if metadata and metadata.pyproject_version:
|
|
177
|
-
return metadata.pyproject_version
|
|
178
|
-
|
|
179
175
|
for env_var in ("CALGITVER", "GIT_COMMIT"):
|
|
180
176
|
env_var_version = os.getenv(env_var)
|
|
181
177
|
lvl = LOGGER.debug if env_var == "CALGITVER" else LOGGER.info
|
|
@@ -240,16 +236,6 @@ def get_commit(pkg: Package = "") -> str: # should really be named get_commit_h
|
|
|
240
236
|
except git.NO_GIT:
|
|
241
237
|
pass
|
|
242
238
|
|
|
243
|
-
try:
|
|
244
|
-
if pkg:
|
|
245
|
-
LOGGER.debug("`get_commit` reading from metadata.")
|
|
246
|
-
metadata = read_metadata(pkg)
|
|
247
|
-
if metadata.is_empty:
|
|
248
|
-
raise EmptyMetadataException
|
|
249
|
-
return metadata.git_commit
|
|
250
|
-
except EmptyMetadataException:
|
|
251
|
-
pass
|
|
252
|
-
|
|
253
239
|
LOGGER.warning("`get_commit` found no commit.")
|
|
254
240
|
return ""
|
|
255
241
|
|
|
@@ -269,16 +255,6 @@ def is_clean(pkg: Package = "") -> bool:
|
|
|
269
255
|
except git.NO_GIT:
|
|
270
256
|
pass
|
|
271
257
|
|
|
272
|
-
try:
|
|
273
|
-
if pkg:
|
|
274
|
-
LOGGER.debug("`is_clean` reading from metadata.")
|
|
275
|
-
metadata = read_metadata(pkg)
|
|
276
|
-
if metadata.is_empty:
|
|
277
|
-
raise EmptyMetadataException
|
|
278
|
-
return metadata.git_is_clean
|
|
279
|
-
except EmptyMetadataException:
|
|
280
|
-
pass
|
|
281
|
-
|
|
282
258
|
LOGGER.warning("`is_clean` found no cleanliness - assume dirty.")
|
|
283
259
|
return False
|
|
284
260
|
|
|
@@ -294,16 +270,6 @@ def get_branch(pkg: Package = "", format: NameFormatType = "git") -> str:
|
|
|
294
270
|
except git.NO_GIT:
|
|
295
271
|
pass
|
|
296
272
|
|
|
297
|
-
try:
|
|
298
|
-
if pkg:
|
|
299
|
-
LOGGER.debug("`get_branch` reading from metadata.")
|
|
300
|
-
metadata = read_metadata(pkg)
|
|
301
|
-
if not metadata.git_branch:
|
|
302
|
-
raise EmptyMetadataException
|
|
303
|
-
return metadata.git_branch
|
|
304
|
-
except EmptyMetadataException:
|
|
305
|
-
pass
|
|
306
|
-
|
|
307
273
|
LOGGER.warning("`get_branch` found no branch.")
|
|
308
274
|
return ""
|
|
309
275
|
|
|
@@ -316,27 +282,12 @@ def get_user(pkg: Package = "", format: NameFormatType = "git") -> str:
|
|
|
316
282
|
LOGGER.debug("`get_user` reading from env var.")
|
|
317
283
|
return os.environ[THDS_USER]
|
|
318
284
|
|
|
319
|
-
try:
|
|
320
|
-
if pkg:
|
|
321
|
-
LOGGER.debug("`get_user` reading from metadata.")
|
|
322
|
-
metadata = read_metadata(pkg)
|
|
323
|
-
if not metadata.thds_user:
|
|
324
|
-
raise EmptyMetadataException
|
|
325
|
-
return metadata.thds_user
|
|
326
|
-
except EmptyMetadataException:
|
|
327
|
-
pass
|
|
328
|
-
|
|
329
285
|
LOGGER.debug("`get_user` found no user data - getting system user.")
|
|
330
286
|
return getuser()
|
|
331
287
|
|
|
332
288
|
return format_name(_get_user(pkg), format)
|
|
333
289
|
|
|
334
290
|
|
|
335
|
-
def is_deployed(pkg: Package) -> bool:
|
|
336
|
-
meta = read_metadata(pkg)
|
|
337
|
-
return not meta.is_empty
|
|
338
|
-
|
|
339
|
-
|
|
340
291
|
def _hacky_get_pyproject_toml_version(pkg: Package, wdir: Path) -> str:
|
|
341
292
|
# it will be a good day when Python packages a toml reader by default.
|
|
342
293
|
ppt = wdir / "pyproject.toml"
|
|
@@ -379,14 +330,14 @@ def find_pyproject_toml_version(starting_path: Path, pkg: Package) -> str:
|
|
|
379
330
|
MiscType = ty.Mapping[str, ty.Union[str, int, float, bool]]
|
|
380
331
|
|
|
381
332
|
|
|
382
|
-
@
|
|
333
|
+
@dataclass(frozen=True)
|
|
383
334
|
class Metadata:
|
|
384
335
|
git_commit: str = ""
|
|
385
336
|
git_branch: str = ""
|
|
386
337
|
git_is_clean: bool = False
|
|
387
338
|
pyproject_version: str = "" # only present if the project defines `version` inside pyproject.toml
|
|
388
339
|
thds_user: str = ""
|
|
389
|
-
misc: MiscType =
|
|
340
|
+
misc: MiscType = field(default_factory=lambda: MappingProxyType(dict()))
|
|
390
341
|
|
|
391
342
|
@property
|
|
392
343
|
def docker_branch(self) -> str:
|
|
@@ -406,103 +357,13 @@ class Metadata:
|
|
|
406
357
|
|
|
407
358
|
@property
|
|
408
359
|
def is_empty(self) -> bool:
|
|
409
|
-
return all(not getattr(self,
|
|
360
|
+
return all(not getattr(self, f.name) for f in self.__dataclass_fields__.values())
|
|
410
361
|
|
|
411
362
|
@property
|
|
412
363
|
def git_is_dirty(self) -> bool:
|
|
413
364
|
return not self.git_is_clean
|
|
414
365
|
|
|
415
366
|
|
|
416
|
-
meta_converter = Converter(forbid_extra_keys=True)
|
|
417
|
-
meta_converter.register_structure_hook(
|
|
418
|
-
Metadata, lambda v, _: Metadata(misc=MappingProxyType(v.pop("misc", {})), **v)
|
|
419
|
-
)
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
class EmptyMetadataException(Exception):
|
|
423
|
-
pass
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
def init_metadata(misc: ty.Optional[MiscType] = None, pyproject_toml_version: str = "") -> Metadata:
|
|
427
|
-
return Metadata(
|
|
428
|
-
git_commit=get_commit(),
|
|
429
|
-
git_branch=get_branch(),
|
|
430
|
-
git_is_clean=is_clean(),
|
|
431
|
-
pyproject_version=pyproject_toml_version,
|
|
432
|
-
thds_user=os.getenv(THDS_USER, getuser()),
|
|
433
|
-
misc=MappingProxyType(misc) if misc else MappingProxyType(dict()),
|
|
434
|
-
)
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
def _sanitize_metadata_for_docker_tools(d: dict):
|
|
438
|
-
"""We want our Docker builds to be able to take advantage of
|
|
439
|
-
caching based on the contents of the sources copied over into
|
|
440
|
-
them. If we embed a meta.json into each library where the commit
|
|
441
|
-
hash changes every time a commit happens, then we've blown away
|
|
442
|
-
our entire cache.
|
|
443
|
-
|
|
444
|
-
The Docker builds already inject this metadata as environment
|
|
445
|
-
variables after the source copies happen, so there's no need for
|
|
446
|
-
us to embed it this way.
|
|
447
|
-
"""
|
|
448
|
-
d["git_commit"] = ""
|
|
449
|
-
d["git_branch"] = ""
|
|
450
|
-
d["git_is_clean"] = ""
|
|
451
|
-
d["thds_user"] = THDS_USER
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
def write_metadata(
|
|
455
|
-
pkg: str,
|
|
456
|
-
*,
|
|
457
|
-
misc: ty.Optional[MiscType] = None,
|
|
458
|
-
namespace: str = "thds",
|
|
459
|
-
layout: LayoutType = "src",
|
|
460
|
-
wdir: ty.Optional[StrOrPath] = None,
|
|
461
|
-
deploying: bool = False,
|
|
462
|
-
for_docker_tools_build: bool = False,
|
|
463
|
-
) -> None:
|
|
464
|
-
wdir_ = Path(wdir) if wdir else Path(".")
|
|
465
|
-
assert wdir_
|
|
466
|
-
if os.getenv(DEPLOYING) or deploying:
|
|
467
|
-
LOGGER.debug("Writing metadata.")
|
|
468
|
-
metadata = init_metadata(
|
|
469
|
-
misc=misc, pyproject_toml_version=_hacky_get_pyproject_toml_version(pkg, wdir_)
|
|
470
|
-
)
|
|
471
|
-
metadata_path = os.path.join(
|
|
472
|
-
"src" if layout == "src" else "",
|
|
473
|
-
namespace.replace("-", "/").replace(".", "/"),
|
|
474
|
-
pkg.replace("-", "_").replace(".", "/"),
|
|
475
|
-
META_FILE,
|
|
476
|
-
)
|
|
477
|
-
|
|
478
|
-
LOGGER.info(f"Writing metadata for {pkg} to {wdir_ / metadata_path}")
|
|
479
|
-
with open(wdir_ / metadata_path, "w") as f:
|
|
480
|
-
metadata_dict = meta_converter.unstructure(metadata)
|
|
481
|
-
if for_docker_tools_build:
|
|
482
|
-
_sanitize_metadata_for_docker_tools(metadata_dict)
|
|
483
|
-
json.dump(metadata_dict, f, indent=2)
|
|
484
|
-
f.write("\n") # Add newline because Py JSON does not
|
|
485
|
-
|
|
486
|
-
|
|
487
367
|
@lru_cache(None)
|
|
488
368
|
def read_metadata(pkg: Package) -> Metadata:
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
if pkg == "__main__":
|
|
492
|
-
raise ValueError("`read_meta` expects a package or module name, not '__main__'.")
|
|
493
|
-
|
|
494
|
-
if not pkg:
|
|
495
|
-
raise ValueError(
|
|
496
|
-
"`read_meta` is missing a package or module name. "
|
|
497
|
-
"If using `__package__` make sure an __init__.py is present."
|
|
498
|
-
)
|
|
499
|
-
|
|
500
|
-
try:
|
|
501
|
-
with open_text(pkg, META_FILE) as f:
|
|
502
|
-
return meta_converter.structure(json.load(f), Metadata)
|
|
503
|
-
# pkg=__name__ will raise a TypeError unless it is called in an __init__.py
|
|
504
|
-
except (ModuleNotFoundError, FileNotFoundError, TypeError):
|
|
505
|
-
pkg_ = pkg.split(".")
|
|
506
|
-
if len(pkg_) <= 1:
|
|
507
|
-
return Metadata()
|
|
508
|
-
return read_metadata(".".join(pkg_[:-1]))
|
|
369
|
+
return Metadata() # this is now deprecated because we don't use it.
|
thds/core/sqlite/insert_utils.py
CHANGED
|
@@ -19,13 +19,13 @@ thds/core/hashing.py,sha256=OqaV65vGKpT3l78jm-Uh7xG4DtAczGjk9-Q60OGmhY0,3521
|
|
|
19
19
|
thds/core/home.py,sha256=tTClL_AarIKeri1aNCpuIC6evD7qr83ESGD173B81hU,470
|
|
20
20
|
thds/core/hostname.py,sha256=canFGr-JaaG7nUfsQlyL0JT-2tnZoT1BvXzyaOMK1vA,208
|
|
21
21
|
thds/core/imports.py,sha256=0LVegY8I8_XKZPcqiIp2OVVzEDtyqYA3JETf9OAKNKs,568
|
|
22
|
-
thds/core/inspect.py,sha256=
|
|
22
|
+
thds/core/inspect.py,sha256=5Gckjux-nc8cz7OS6Go9Bmx2n_wWmGr42gVRuysAWFw,1985
|
|
23
23
|
thds/core/iterators.py,sha256=d3iTQDR0gCW1nMRmknQeodR_4THzR9Ajmp8F8KCCFgg,208
|
|
24
24
|
thds/core/lazy.py,sha256=e1WvG4LsbEydV0igEr_Vl1cq05zlQNIE8MFYT90yglE,3289
|
|
25
25
|
thds/core/link.py,sha256=kmFJIFvEZc16-7S7IGvtTpzwl3VuvFl3yPlE6WJJ03w,5404
|
|
26
26
|
thds/core/logical_root.py,sha256=gWkIYRv9kNQfzbpxJaYiwNXVz1neZ2NvnvProtOn9d8,1399
|
|
27
27
|
thds/core/merge_args.py,sha256=7oj7dtO1-XVkfTM3aBlq3QlZbo8tb6X7E3EVIR-60t8,5781
|
|
28
|
-
thds/core/meta.py,sha256=
|
|
28
|
+
thds/core/meta.py,sha256=BQls5UQpdVEd6mDlkYk8_c-k_OYkpvde9Doa_ATSrfA,12067
|
|
29
29
|
thds/core/parallel.py,sha256=HXAn9aIYqNE5rnRN5ypxR6CUucdfzE5T5rJ_MUv-pFk,7590
|
|
30
30
|
thds/core/pickle_visit.py,sha256=QNMWIi5buvk2zsvx1-D-FKL7tkrFUFDs387vxgGebgU,833
|
|
31
31
|
thds/core/prof.py,sha256=5ViolfPsAPwUTHuhAe-bon7IArPGXydpGoB5uZmObDk,8264
|
|
@@ -59,7 +59,7 @@ thds/core/sqlite/copy.py,sha256=y3IRQTBrWDfKuVIfW7fYuEgwRCRKHjN0rxVFkIb9VrQ,1155
|
|
|
59
59
|
thds/core/sqlite/ddl.py,sha256=k9BvmDzb0rrlhmEpXkB6ESaZAUWtbL58x-70sPyoFk4,201
|
|
60
60
|
thds/core/sqlite/functions.py,sha256=AOIRzb7lNxmFm1J5JS6R8Nl-dSv3Dy47UNZVVjl1rvk,2158
|
|
61
61
|
thds/core/sqlite/index.py,sha256=Vc7qxPqQ69A6GO5gmVQf5e3y8f8IqOTHgyEDoVZxTFM,903
|
|
62
|
-
thds/core/sqlite/insert_utils.py,sha256=
|
|
62
|
+
thds/core/sqlite/insert_utils.py,sha256=BNI3VUdqwBdaqa0xqiJrhE6XyzPsTF8N4KKKdb4Vfes,884
|
|
63
63
|
thds/core/sqlite/merge.py,sha256=NxettDMJ_mcrWfteQn_ERY7MUB5ETR-yJLKg7uvF6zA,3779
|
|
64
64
|
thds/core/sqlite/meta.py,sha256=4P65PAmCjagHYO1Z6nWM-wkjEWv3hxw5qVa4cIpcH_8,5859
|
|
65
65
|
thds/core/sqlite/read.py,sha256=5pWvrbed3XNWgSy-79-8ONWkkt4jWbTzFNW6SnOrdYQ,2576
|
|
@@ -68,8 +68,8 @@ thds/core/sqlite/structured.py,sha256=SvZ67KcVcVdmpR52JSd52vMTW2ALUXmlHEeD-VrzWV
|
|
|
68
68
|
thds/core/sqlite/types.py,sha256=oUkfoKRYNGDPZRk29s09rc9ha3SCk2SKr_K6WKebBFs,1308
|
|
69
69
|
thds/core/sqlite/upsert.py,sha256=BmKK6fsGVedt43iY-Lp7dnAu8aJ1e9CYlPVEQR2pMj4,5827
|
|
70
70
|
thds/core/sqlite/write.py,sha256=z0219vDkQDCnsV0WLvsj94keItr7H4j7Y_evbcoBrWU,3458
|
|
71
|
-
thds_core-1.38.
|
|
72
|
-
thds_core-1.38.
|
|
73
|
-
thds_core-1.38.
|
|
74
|
-
thds_core-1.38.
|
|
75
|
-
thds_core-1.38.
|
|
71
|
+
thds_core-1.38.20250618170443.dist-info/METADATA,sha256=fMcbZQryaflWaA_t75netjtnOsKfNpANLmLaQ3UJuJA,2216
|
|
72
|
+
thds_core-1.38.20250618170443.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
73
|
+
thds_core-1.38.20250618170443.dist-info/entry_points.txt,sha256=bOCOVhKZv7azF3FvaWX6uxE6yrjK6FcjqhtxXvLiFY8,161
|
|
74
|
+
thds_core-1.38.20250618170443.dist-info/top_level.txt,sha256=LTZaE5SkWJwv9bwOlMbIhiS-JWQEEIcjVYnJrt-CriY,5
|
|
75
|
+
thds_core-1.38.20250618170443.dist-info/RECORD,,
|
|
File without changes
|
{thds_core-1.38.20250617225213.dist-info → thds_core-1.38.20250618170443.dist-info}/entry_points.txt
RENAMED
|
File without changes
|
{thds_core-1.38.20250617225213.dist-info → thds_core-1.38.20250618170443.dist-info}/top_level.txt
RENAMED
|
File without changes
|