xedocs 0.2.26__tar.gz → 0.2.28__tar.gz

xedocs-0.2.28/AUTHORS.rst

@@ -0,0 +1,13 @@
+=======
+Credits
+=======
+
+Development Lead
+----------------
+
+* Yossi Mosbacher <joe.mosbacher@gmail.com>
+
+Contributors
+------------
+
+None yet. Why not be the first?

{xedocs-0.2.26 → xedocs-0.2.28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: xedocs
-Version: 0.2.26
+Version: 0.2.28
 Summary: Top-level package for xedocs.
 Home-page: https://github.com/XENONnT/xedocs
 Author: Yossi Mosbacher
@@ -32,8 +32,10 @@ Description-Content-Type: text/x-rst
 =======================================
 XeDocs - XENON Metadata management tool
 =======================================
-xedocs is meant to replace cmt and bodega as well as helping tracking all shared documents especially if
-they need to be versioned.
+
+XeDocs manages tracking versioned detector numbers, replacing CMT and ideally all hard-coded values. 
+XeDocs both looks up data from its own online database, and uses straxen URL-style lookup to find other resources. 
+To upload data to the XeDocs database, you must submit it as a PR to https://github.com/XENONnT/corrections
 
 What does Xedocs give you
 =========================

{xedocs-0.2.26 → xedocs-0.2.28}/README.rst

@@ -1,8 +1,10 @@
 =======================================
 XeDocs - XENON Metadata management tool
 =======================================
-xedocs is meant to replace cmt and bodega as well as helping tracking all shared documents especially if
-they need to be versioned.
+
+XeDocs manages tracking versioned detector numbers, replacing CMT and ideally all hard-coded values. 
+XeDocs both looks up data from its own online database, and uses straxen URL-style lookup to find other resources. 
+To upload data to the XeDocs database, you must submit it as a PR to https://github.com/XENONnT/corrections
 
 What does Xedocs give you
 =========================

{xedocs-0.2.26 → xedocs-0.2.28}/pyproject.toml

@@ -1,7 +1,7 @@
 [tool]
 [tool.poetry]
 name = "xedocs"
-version = "0.2.26"
+version = "0.2.28"
 homepage = "https://github.com/XENONnT/xedocs"
 description = "Top-level package for xedocs."
 authors = ["Yossi Mosbacher <joe.mosbacher@gmail.com>"]

{xedocs-0.2.26 → xedocs-0.2.28}/xedocs/__init__.py

@@ -2,7 +2,7 @@
 
 __author__ = """Yossi Mosbacher"""
 __email__ = "joe.mosbacher@gmail.com"
-__version__ = "0.2.26"
+__version__ = "0.2.28"
 
 
 import logging

{xedocs-0.2.26 → xedocs-0.2.28}/xedocs/data_locations/mongodb.py

@@ -5,9 +5,9 @@ from rframe import DataAccessor
 
 
 def xenon_config_source(settings: BaseSettings) -> Dict[str, Any]:
-    from xedocs import settings
+    from xedocs import settings as xenon_settings
 
-    cfg = settings.xenon_config.RunDB
+    cfg = xenon_settings.xenon_config.RunDB
     data = {}
     if cfg.xent_url:
         data['host'] = cfg.xent_url
@@ -17,6 +17,14 @@ def xenon_config_source(settings: BaseSettings) -> Dict[str, Any]:
         data['password'] = cfg.xent_password
     if cfg.xent_database:
         data['auth_db'] = cfg.xent_database
+    if cfg.max_pool_size:
+        data['max_pool_size'] = cfg.max_pool_size
+    if cfg.socket_timeout:
+        data['socket_timeout'] = cfg.socket_timeout
+    if cfg.connect_timeout:
+        data['connect_timeout'] = cfg.connect_timeout
+    if cfg.read_preference:
+        data['read_preference'] = cfg.read_preference
     return data
 
 
@@ -50,17 +58,26 @@ class MongoDB(BaseSettings):
     host: str = "localhost"
     connection_uri: str = None
 
+    max_pool_size: int = 100
+    socket_timeout: int = 60000
+    connect_timeout: int = 60000
+    read_preference: str = "secondaryPreferred"
+
     @property
     def client(self):
         if self.connection_uri is None:
-            self.connection_uri = f"mongodb://{self.username}:{self.password}@{self.host}?authSource={self.auth_db}&readPreference=secondary"
+            self.connection_uri = f"mongodb://{self.username}:{self.password}@{self.host}?authSource={self.auth_db}"
         if self.connection_uri not in self.CLIENT_CACHE:
             self.CLIENT_CACHE[self.connection_uri] = self.make_client(self.connection_uri)
         return self.CLIENT_CACHE[self.connection_uri]
 
     def make_client(self, connection_uri):
         import pymongo
-        return pymongo.MongoClient(connection_uri)
+        return pymongo.MongoClient(connection_uri, 
+                                   maxPoolSize=self.max_pool_size,
+                                   socketTimeoutMS=self.socket_timeout,
+                                   connectTimeoutMS=self.connect_timeout,
+                                   readPreference=self.read_preference)
 
     @classmethod
     def from_utilix(cls, **kwargs):

xedocs-0.2.28/xedocs/schemas/corrections/__init__.py

@@ -0,0 +1,3 @@
+from .implementations import *
+from .base_corrections import *
+from .base_references import *

{xedocs-0.2.26/xedocs/schemas/corrections → xedocs-0.2.28/xedocs/schemas/corrections/implementations}/__init__.py

@@ -1,5 +1,3 @@
-from .base_corrections import *
-from .base_references import *
 from .electron_drift import *
 from .electron_lifetime import *
 from .extraction_efficiency import *
@@ -26,3 +24,4 @@ from .s2_pattern_maps import *
 from .s1_pattern_maps import *
 from .z_bias import *
 from .cs2_aft_corrections import *
+from .se_gain_partition import *

{xedocs-0.2.26/xedocs/schemas/corrections → xedocs-0.2.28/xedocs/schemas/corrections/implementations}/avg_se_gain.py

@@ -9,11 +9,11 @@ wiki:
 
 import rframe
 
-from .base_corrections import TimeSampledCorrection
-from ..constants import PARTITION
+from ..base_corrections import TimeIntervalCorrection
+from ...constants import PARTITION
 
 
-class AvgSingleElectronGain(TimeSampledCorrection):
+class AvgSingleElectronGain(TimeIntervalCorrection):
 
     _ALIAS = "avg_se_gains"
     partition: PARTITION = rframe.Index(default="all_tpc")

{xedocs-0.2.26/xedocs/schemas/corrections → xedocs-0.2.28/xedocs/schemas/corrections/implementations}/baseline_samples_nv.py

@@ -8,7 +8,7 @@ wiki:
 """
 
 
-from .base_corrections import TimeSampledCorrection
+from ..base_corrections import TimeSampledCorrection
 
 
 class BaselineSamplesNV(TimeSampledCorrection):

{xedocs-0.2.26/xedocs/schemas/corrections → xedocs-0.2.28/xedocs/schemas/corrections/implementations}/bayes_model.py

@@ -5,7 +5,7 @@ import datetime
 from typing import Literal
 import rframe
 
-from .base_references import BaseResourceReference
+from ..base_references import BaseResourceReference
 
 
 class NaiveBayesClassifier(BaseResourceReference):

{xedocs-0.2.26/xedocs/schemas/corrections → xedocs-0.2.28/xedocs/schemas/corrections/implementations}/cs2_aft_corrections.py

@@ -6,7 +6,7 @@ https://xe1t-wiki.lngs.infn.it/doku.php?id=xenon:xenonnt:zihao:sr1_s2aft_photoni
 
 """
 
-from .base_corrections import TimeSampledCorrection
+from ..base_corrections import TimeSampledCorrection
 
 
 class cS2BottomTopRatioCorrection(TimeSampledCorrection):

{xedocs-0.2.26/xedocs/schemas/corrections → xedocs-0.2.28/xedocs/schemas/corrections/implementations}/electron_diffusion_cte.py

@@ -7,7 +7,7 @@
 
 import rframe
 
-from .base_corrections import TimeSampledCorrection
+from ..base_corrections import TimeSampledCorrection
 
 
 class ElectronDiffusionCte(TimeSampledCorrection):

{xedocs-0.2.26/xedocs/schemas/corrections → xedocs-0.2.28/xedocs/schemas/corrections/implementations}/electron_drift.py

@@ -13,7 +13,7 @@ See note on drift velocity, diffusion constant, and drift time at the gate
 
 import rframe
 
-from .base_corrections import TimeSampledCorrection
+from ..base_corrections import TimeSampledCorrection
 
 
 class ElectronDriftVelocity(TimeSampledCorrection):

{xedocs-0.2.26/xedocs/schemas/corrections → xedocs-0.2.28/xedocs/schemas/corrections/implementations}/electron_lifetime.py

@@ -8,7 +8,7 @@ See [description in the Team C overview page](https://xe1t-wiki.lngs.infn.it/dok
 
 """
 
-from .base_corrections import TimeSampledCorrection
+from ..base_corrections import TimeSampledCorrection
 
 
 class ElectronLifetime(TimeSampledCorrection):

{xedocs-0.2.26/xedocs/schemas/corrections → xedocs-0.2.28/xedocs/schemas/corrections/implementations}/extraction_efficiency.py

@@ -13,8 +13,8 @@ Time dependence of Extraction Efficiency generated from Kr83m calibration data a
 """
 import rframe
 
-from .base_corrections import TimeSampledCorrection
-from ..constants import PARTITION
+from ..base_corrections import TimeSampledCorrection
+from ...constants import PARTITION
 
 
 class RelExtractionEff(TimeSampledCorrection):

{xedocs-0.2.26/xedocs/schemas/corrections → xedocs-0.2.28/xedocs/schemas/corrections/implementations}/field_distortion.py

@@ -9,7 +9,7 @@ See [description in the Team C overview page](https://xe1t-wiki.lngs.infn.it/dok
 
 """
 
-from .base_references import BaseMap
+from ..base_references import BaseMap
 
 
 class FdcMap(BaseMap):

{xedocs-0.2.26/xedocs/schemas/corrections → xedocs-0.2.28/xedocs/schemas/corrections/implementations}/global_versions.py

@@ -1,6 +1,6 @@
 import rframe
 
-from .base_corrections import TimeIntervalCorrection
+from ..base_corrections import TimeIntervalCorrection
 
 
 class GlobalVersion(TimeIntervalCorrection):

{xedocs-0.2.26/xedocs/schemas/corrections → xedocs-0.2.28/xedocs/schemas/corrections/implementations}/hit_thresholds.py

@@ -6,8 +6,8 @@
 
 import rframe
 
-from .base_corrections import TimeSampledCorrection
-from ..constants import DETECTOR
+from ..base_corrections import TimeSampledCorrection
+from ...constants import DETECTOR
 
 
 class HitThreshold(TimeSampledCorrection):

{xedocs-0.2.26/xedocs/schemas/corrections → xedocs-0.2.28/xedocs/schemas/corrections/implementations}/hotspot_veto_cut.py

@@ -4,7 +4,7 @@ which vetos the events with local single electron rate above the threshold in ea
 Reference: xenon:xenonnt:analysis:hot_spot_cut_summary
 """
 
-from .base_corrections import TimeIntervalCorrection
+from ..base_corrections import TimeIntervalCorrection
 
 class HotspotVetoThreshold(TimeIntervalCorrection):
     _ALIAS = 'hotspot_veto_thresholds'

{xedocs-0.2.26/xedocs/schemas/corrections → xedocs-0.2.28/xedocs/schemas/corrections/implementations}/photoionization_strength.py

@@ -3,7 +3,7 @@
 https://xe1t-wiki.lngs.infn.it/doku.php?id=xenon:xenonnt_sr1:photoionization_origin
 """
 
-from .base_corrections import TimeSampledCorrection
+from ..base_corrections import TimeSampledCorrection
 
 
 class PhotoionizationStrength(TimeSampledCorrection):

{xedocs-0.2.26/xedocs/schemas/corrections → xedocs-0.2.28/xedocs/schemas/corrections/implementations}/pmt_area_to_pe.py

@@ -8,8 +8,8 @@
 
 import rframe
 
-from .base_corrections import TimeSampledCorrection
-from ..constants import DETECTOR
+from ..base_corrections import TimeSampledCorrection
+from ...constants import DETECTOR
 
 
 class PmtAreaToPE(TimeSampledCorrection):

{xedocs-0.2.26/xedocs/schemas/corrections → xedocs-0.2.28/xedocs/schemas/corrections/implementations}/position_reconstruction.py

@@ -16,7 +16,7 @@ from typing import Literal
 
 import rframe
 
-from .base_references import BaseResourceReference
+from ..base_references import BaseResourceReference
 
 
 class PosRecModel(BaseResourceReference):

{xedocs-0.2.26/xedocs/schemas/corrections → xedocs-0.2.28/xedocs/schemas/corrections/implementations}/relative_light_yield.py

@@ -11,7 +11,7 @@
 
 import rframe
 
-from .base_corrections import TimeSampledCorrection
+from ..base_corrections import TimeSampledCorrection
 
 
 class RelativeLightYield(TimeSampledCorrection):

{xedocs-0.2.26/xedocs/schemas/corrections → xedocs-0.2.28/xedocs/schemas/corrections/implementations}/s1_aft_xyz.py

@@ -6,7 +6,7 @@ Should be treated like a normal correction for dependencies and should be update
 
 """
 
-from .base_references import BaseResourceReference
+from ..base_references import BaseResourceReference
 
 
 class S1AFTXYZMap(BaseResourceReference):

{xedocs-0.2.26/xedocs/schemas/corrections → xedocs-0.2.28/xedocs/schemas/corrections/implementations}/s1_pattern_maps.py

@@ -5,7 +5,7 @@ https://xe1t-wiki.lngs.infn.it/doku.php?id=min:s1opticalmap_comparison (Min + Gi
 
 """
 
-from .base_references import BaseMap
+from ..base_references import BaseMap
 
 
 class S1PatternMap(BaseMap):

{xedocs-0.2.26/xedocs/schemas/corrections → xedocs-0.2.28/xedocs/schemas/corrections/implementations}/s1_xyz.py

@@ -8,7 +8,7 @@ The jupyter notebook in this folder is replaced by [this](https://github.com/XEN
 
 """
 
-from .base_references import BaseMap
+from ..base_references import BaseMap
 
 
 

{xedocs-0.2.26/xedocs/schemas/corrections → xedocs-0.2.28/xedocs/schemas/corrections/implementations}/s2_pattern_maps.py

@@ -15,7 +15,7 @@ https://xe1t-wiki.lngs.infn.it/doku.php?id=xenon:xenon1t:jacques:nt_sr1_s2pl (Ja
 
 """
 
-from .base_references import BaseMap
+from ..base_references import BaseMap
 
 
 class S2PatternMap(BaseMap):

{xedocs-0.2.26/xedocs/schemas/corrections → xedocs-0.2.28/xedocs/schemas/corrections/implementations}/s2_xy.py

@@ -19,7 +19,7 @@ See [description in the Team C overview page](https://xe1t-wiki.lngs.infn.it/dok
 
 """
 
-from .base_references import BaseMap
+from ..base_references import BaseMap
 
 
 class S2XYMap(BaseMap):

xedocs-0.2.28/xedocs/schemas/corrections/implementations/se_gain_partition.py

@@ -0,0 +1,21 @@
+"""
+Correction: Region linear and circular for AB/CD partitions
+Affects: Corrected areas
+
+Two distinct patterns of evolution of single electron corrections between A+B and C+D. Distinguish thanks to linear and circular regions.
+This TPC partitioning affects the single electron gains - both the average value and not - and the relative extraction efficiency.
+
+SR0 wiki: https://xe1t-wiki.lngs.infn.it/doku.php?id=jlong:sr0_2_region_se_correction
+SR1 wiki: https://xe1t-wiki.lngs.infn.it/doku.php?id=xenon:xenonnt:noahhood:corrections:se_gain_ee_final
+"""
+
+import rframe
+
+from ..base_corrections import TimeIntervalCorrection
+
+class SingleElectronGainPartition(TimeIntervalCorrection):
+
+    _ALIAS = "single_electron_gain_partition"
+    region: str = rframe.Index(max_length=80)
+    value: float
+

{xedocs-0.2.26/xedocs/schemas/corrections → xedocs-0.2.28/xedocs/schemas/corrections/implementations}/se_gains.py

@@ -13,8 +13,8 @@ Wiki:
 
 import rframe
 
-from .base_corrections import TimeSampledCorrection
-from ..constants import PARTITION
+from ..base_corrections import TimeSampledCorrection
+from ...constants import PARTITION
 
 
 class SingleElectronGain(TimeSampledCorrection):

{xedocs-0.2.26/xedocs/schemas/corrections → xedocs-0.2.28/xedocs/schemas/corrections/implementations}/som_network.py

@@ -4,7 +4,7 @@ Self-Organizing Map to classify peaklets as one of several SOM_types,
 as well as classifying those types into either an S1 or an S2
 """
 
-from .base_references import BaseResourceReference
+from ..base_references import BaseResourceReference
 
 
 class SOMClassifier(BaseResourceReference):

{xedocs-0.2.26/xedocs/schemas/corrections → xedocs-0.2.28/xedocs/schemas/corrections/implementations}/z_bias.py

@@ -5,7 +5,7 @@ the observed r(r_obs) and z(z_obs).
 Reference: xenon:xenonnt:terliuk:drift_field_z_bias_correction
 """
 
-from .base_references import BaseResourceReference
+from ..base_references import BaseResourceReference
 
 class ZBias(BaseResourceReference):
     _ALIAS = 'z_bias_maps'

{xedocs-0.2.26 → xedocs-0.2.28}/xedocs/schemas/detector_numbers.py

@@ -1,11 +1,10 @@
 import rframe
 import datetime
 
-from .base_schemas import VersionedXeDoc
+from .corrections.base_corrections import TimeIntervalCorrection
 from .constants import PARTITION
 
-
-class DetectorNumber(VersionedXeDoc):
+class DetectorNumber(TimeIntervalCorrection):
     """Detector parameters
     A collection of non-time dependent detector
     values.
@@ -15,10 +14,8 @@ class DetectorNumber(VersionedXeDoc):
 
     field: str = rframe.Index(max_length=80)
     partition: PARTITION = rframe.Index(default="all_tpc")
-
+    science_run: str = rframe.Index(max_length=80)
+    definition: str=""
     value: float
     uncertainty: float
-    definition: str
     reference: str = ""
-    date: datetime.datetime
-    comments: str = ""

{xedocs-0.2.26 → xedocs-0.2.28}/xedocs/xedocs.py

@@ -10,6 +10,7 @@ from tqdm.auto import tqdm
 
 from ._settings import settings
 from .schemas import XeDoc
+from .data_locations.mongodb import MongoDB
 
 
 def find_docs(schema, datasource=None, **labels):
@@ -77,6 +78,40 @@ def find_one(schema, datasource=None, **labels):
 
 
 def insert_docs(schema: str, docs: Union[list, dict, pd.DataFrame], datasource=None, dry=False):
+    # Currently stuck on how to deal with instances of schemas
+    if datasource == 'straxen_db': # switch to straxen_db
+        mongo_username = MongoDB().username
+        if mongo_username == 'corrections_expert':
+            # If statements only trigger
+            target_version = "ONLINE"
+            # Note to self: This is done in a very dumb way, fix later
+            ONLINE_check = True
+            if isinstance(docs, pd.DataFrame):
+                ONLINE_check = (docs['version'] == target_version).all()
+
+            elif isinstance(docs, dict):
+                ONLINE_check = all(item['version'] == target_version for item in docs)
+
+            else:
+                if isinstance(docs, list):
+                    if isinstance(docs[0], list) or isinstance(docs[0], dict):
+                        # It could be a list of dicts, or list or schemas...
+                        # This can be an actual list or a list of docs
+                        # these cannot be treated the same
+                        ONLINE_check = all(item['version'] == target_version for item in docs)
+                    else:
+                        # This assumes the last choice is a schema, if not other things will yield errors?
+                        # This is kinda sloppy...
+                        ONLINE_check = all(item.version == target_version for item in docs)
+                elif hasattr(docs, 'version'):
+                    if docs.version != target_version: # if version isnt ONLINE
+                        ONLINE_check = False
+                else:
+                    ONLINE_check = all(item.version == target_version for item in docs)
+
+            if not ONLINE_check:
+                raise ValueError("You are attempting to modify the a straxen_db correction whose version is not ONLINE")
+
     if isinstance(docs, pd.DataFrame):
         docs = docs.reset_index().to_dict(orient="records")
     if not isinstance(docs, list):
@@ -125,13 +160,13 @@ def find_schema(name) -> Type[XeDoc]:
 
 def get_accessor(schema, db=None):
     import xedocs
-    
+
     schema = find_schema(schema)
     if not issubclass(schema, XeDoc):
         raise TypeError(
             "Schema must be a subclass of XeDoc" "or the name of a known schema."
         )
-    
+
     if db is None:
         db = settings.DEFAULT_DATABASE
     if isinstance(db, str):
@@ -169,7 +204,7 @@ def sync_dbs(from_db, to_db, schemas=None, dry=False):
 
     if isinstance(from_db, str):
         from_db = getattr(xedocs.databases, from_db)()
-    
+
     if isinstance(to_db, str):
         to_db = getattr(xedocs.databases, to_db)()
 

{xedocs-0.2.26 → xedocs-0.2.28}/xedocs/xenon_config.py

@@ -53,6 +53,7 @@ class RunDBConfig(BaseModel):
     max_pool_size: int = 100
     socket_timeout: int = 60000
     connect_timeout: int = 60000
+    read_preference: str = "secondaryPreferred"
 
 
 class StraxenConfig(BaseModel):