rda-python-dbms 1.0.1__py3-none-any.whl

rda_python_dbms/pgddl.py

@@ -0,0 +1,176 @@
+#!/usr/bin/env python3
+#
+##################################################################################
+#
+#     Title : pgddl
+#    Author : Zaihua Ji, zji@ucar.edu
+#      Date : 11/15/2020
+#             2025-04-04 transferred to package rda_python_dbms from
+#             https://github.com/NCAR/rda-database.git
+#   Purpose : process a ddl (Data Definition Language) file to manipulate data
+#             definition of a table defined in tablename.json
+#
+#    Github : https://github.com/NCAR/rda-python-dbms.git
+#
+##################################################################################
+#
+import os
+import sys
+import re
+import pwd
+from os import path as op
+from rda_python_common import PgDBI
+from rda_python_common import PgLOG
+from rda_python_common import PgUtil
+from . import PgDDLLib
+
+PGACT = {
+   'PKY' : ['', 'DEL', 'ADD'],  # DEL - to drop, ADD - to add primary key
+   'UNQ' : ['', 'DEL', 'ADD'],  # DEL - to drop, ADD - to add unique constraint
+   'CMT' : ['', 'DEL', 'ADD'],  # DEL - to set null, ADD to set comment 
+   'DFT' : ['', 'DEL', 'ADD'],  # DEL - to drop, ADD - to set default value
+   'JSN' : ['', 'ADD'],         # ADD - to add a new defination json file
+   'NNL' : ['', 'DEL', 'ADD'],  # DEL - to drop, ADD - to set not null
+   'IDX' : ['', 'DEL', 'ADD', 'CHG'],  # DEL/ADD/CHG - to drop, add and rename index
+   'REF' : ['', 'DEL', 'ADD'],  # DEL - to drop, ADD - to add reference
+   'TBL' : ['', 'DEL', 'ADD'],  # DEL - drop table, ADD - create table
+   'FLD' : ['', 'DEL', 'ADD', 'MOD', 'CHG'],  # ADD/DEL/CHG/MOD - to add, drop, change and modify field
+}
+
+ACTIONS = {
+   'a' : 'ADD',
+   'c' : 'CHG',
+   'd' : 'DEL',
+   'm' : 'MOD'
+}
+
+OPTIONS = {
+   'a' : ['TBL', 'IDX', 'REF'],
+   'f' : 'FLD',
+   'c' : 'CMT',
+   'd' : 'DFT',
+   'j' : 'JSN',
+   'n' : 'NNL',
+   'i' : 'IDX',
+   'p' : 'PKY',
+   'r' : 'REF',
+   't' : 'TBL',
+   'u' : 'UNQ'
+}
+
+VALUES = {
+   'TBL' : [],
+   'IDX' : [],
+   'FLD' : [],
+   'DFT' : [],
+   'NNL' : [],
+   'REF' : [],
+   'UNQ' : []
+}
+
+DBFLDS = {
+   'd' : 'dbname',
+   'c' : 'scname',
+   'h' : 'dbhost',
+   'p' : 'dbport',
+   'u' : 'lnname',
+   'w' : 'pwname',
+}
+
+#
+# main function to excecute this script
+#
+def main():
+
+   option = 't'
+   actopt = False
+   tablenames = action = None
+   dbopt = getall = False
+   argv = sys.argv[1:]
+   for arg in argv:
+      ms = re.match(r'^-([abcdefhlmpstTuwxy])$', arg)
+      if ms:
+         option = ms.group(1)
+         actopt = False
+         if option in 'abefm':
+            if option == "b":
+               PgLOG.PGLOG['BCKGRND'] = 1
+            elif option == "a":
+               getall= True
+            elif option == "e":
+               PgDDLLib.PGDDL['override'] = True
+            elif option == "f":
+               PgDDLLib.PGDDL['usefile'] = True
+            elif option == "m":
+               PgDDLLib.PGDDL['mysqldb'] = True
+            option = None
+         continue
+
+      ms = re.match(r'^-(\w)(\w)$', arg)
+      if ms:
+         action = ms.group(1)
+         option = ms.group(2)
+         actopt = True
+         if option in OPTIONS and action in ACTIONS:
+            act = ACTIONS[action]
+            if option == 'a' and action != 'a':
+               PgLOG.pglog("{}: Invalid Action to {} ALL".format(arg, act), PgLOG.LGEREX)
+            opt = OPTIONS['t'] if option == 'a' else OPTIONS[option]
+            if act in PGACT[opt][1:]:
+               if option == 'a':
+                  for opt in OPTIONS['a']:
+                     PGACT[opt][0] = act
+               else:
+                  PGACT[opt][0] = act
+               if 'ap'.find(option) > -1: option = 't'
+               continue
+         PgLOG.pglog(arg + ": Invalid Action", PgLOG.LGEREX)
+
+      if re.match(r'^-.*', arg): PgLOG.pglog(arg + ": Unknown Option", PgLOG.LGEREX)
+      if not option: PgLOG.pglog(arg + ": Value passed in without leading Option", PgLOG.LGEREX)
+
+      if actopt:
+         VALUES[OPTIONS[option]].append(arg)
+      elif option in DBFLDS:
+         PgDDLLib.DBINFO[DBFLDS[option]] = arg
+         dbopt = True
+      elif option == 'T':
+         PgDDLLib.PGDDL['TBPATH'] = arg
+      elif option == 'l':
+         PgDDLLib.PGDDL['username'] = arg
+      elif option == 'x':
+         PgDDLLib.PGDDL['suffix'].append(arg)
+      elif option == 'y':
+         PgDDLLib.PGDDL['prefix'].append(arg)
+      elif option == 's':
+         PgDBI.PGDBI['SCPATH'] = arg
+      else:
+         VALUES[OPTIONS[option]].append(arg)
+   
+   if dbopt:
+      PgDBI.default_scinfo(PgDDLLib.DBINFO['dbname'], PgDDLLib.DBINFO['scname'], PgDDLLib.DBINFO['dbhost'],
+                           PgDDLLib.DBINFO['lnname'], PgDDLLib.DBINFO['pwname'], PgDDLLib.DBINFO['dbport'])   
+   if VALUES['TBL']:
+      tablenames = VALUES['TBL']
+   elif getall:
+      tablenames = PgDDLLib.allschematables();         # action on all tables
+   if not (tablenames and action): PgLOG.show_usage('pgddl')
+   PgLOG.PGLOG['LOGFILE'] = "pgddl.log"
+   PgLOG.cmdlog("pgddl {}".format(' '.join(argv)))
+
+   # process all or given tables to add/drop table/pkey/index/reference/field
+   act = PGACT['TBL'][0]
+   if act: PgDDLLib.process_tables(tablenames, act, 'TBL')
+   for opt in PGACT:
+      act = PGACT[opt][0]
+      if opt == 'TBL' or not act: continue
+      names = VALUES[opt] if opt in VALUES and VALUES[opt] else None
+      PgDDLLib.process_tables(tablenames, act, opt, names)
+
+   PgLOG.cmdlog()
+   PgLOG.pgexit(0)
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

rda_python_dbms/pgddl.usg

@@ -0,0 +1,59 @@
+
+ Manipulates data definition of PostgreSQL databases.
+
+ Usage: pgddl [-b] [-d DBName] [-c Schema] [-h HostName] [-p DBPort] [-s SchemaPath]   \
+              [-T TBPATH] [-u DBLoginName] [-w DBPassword] [-a] [-e] [-f] [-l username]  \
+              [[-t] TableNamelist] [-x SuffixList] [-y PrefixList] [-xy [names]]
+
+      - Option -b, log process information in logfile only;
+      
+      - Option -c, schema name;
+
+      - Option -d, database name;
+
+      - Option -e, override existing json file for action -aj;
+
+      - Option -f, use .tb files only to add json files for action -aj,
+                   without -f, table info defined in database is used if a
+                   .tb file not exists;
+
+      - Option -h, server host name;
+ 
+      - Option -l, Linux user login name;
+
+      - Option -m, get table info from MySQL server to add json files for action -aj;
+
+      - Option -p, port number to connect to the server remotely;
+ 
+      - Option -s, addtional schema to be include in the schema search_path for
+                   modifying column type;
+ 
+      - Option -t, table name list, the -t can be omitted if table names
+                   are passed in first;
+
+      - Option -T, path to where the table definition files are located;
+
+      - Option -u, database server login name;
+ 
+      - Option -w, user password;
+ 
+      - Option -xy, x = [a]dd, [c]hange, [d]rop, or [m]odify actions on
+           y = [a]ll, [t]able, [p]key, [u]nique, [i]ndex, [r]eference,
+               [j]sonfile, [f]ield, [c]omment, [d]efault, or [n]otnull;
+
+      - Option -a, actions on all existing tables of a given schema;
+
+      - TableNameList, actions described above on given table names;
+
+      - Option -x, suffixes to be appended to the TableNameList with a leading
+        undrescore character, '_', such as TableName_Suffix;
+
+      - Option -y, prefixes to be prepended to the TableNameList with a trailing
+        undrescore character, '_', such as Prefix_TableName;
+
+      One of the actions, -xy, and either -a or TableNameList must be specified
+      to run this application.
+
+ For examples:
+    pgddl -t TableName -aa   # add a table with its fields and associated constraints;
+    pgddl TableName -af FieldName   # add a new Field to a existing table

rda_python_dbms/pgseq.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+#
+##################################################################################
+#
+#     Title : pgseq
+#    Author : Zaihua Ji, zji@ucar.edu
+#      Date : 12/31/2020
+#             2025-04-09 transferred to package rda_python_dbms from
+#             https://github.com/NCAR/rda-database.git
+#   Purpose : check all tables for given schemas to reset the related sequences
+#
+#    Github : https://github.com/NCAR/rda-python-dbms.git
+#
+##################################################################################
+
+import sys
+import re
+from rda_python_common import PgLOG
+from rda_python_common import PgDBI
+from rda_python_common import PgUtil
+
+SINFO = {
+   'ht' : 'rda-pgdb-02.ucar.edu',
+   'db' : 'rdadb',
+   'us' : '',
+   'pw' : '',
+}
+
+#
+# main function to run this program
+#
+def main():
+
+   option = None
+   alltb = 0
+   scnames = []
+   tbnames = []
+   argv = sys.argv[1:]
+
+   for arg in argv:
+      if arg == "-b":
+         PgLOG.MYLOG['BCKGRND'] = 1
+      elif re.match(r'^-([dhpstu])$', arg):
+         option = arg[1]
+      elif re.match(r'^-', arg):
+         PgLOG.pglog(arg + ": Invalid Option", PgLOG.LGWNEX)
+      elif option:
+         if option == 'd':
+            SINFO['db'] = arg
+         elif option == 'h':
+            SINFO['ht'] = arg
+         elif option == 'p':
+            SINFO['pw'] = arg
+         elif option == 'u':
+            SINFO['us'] = arg
+         elif option == 's':
+            scnames.append(arg)
+         elif option == 't':
+            tbnames.append(arg)
+      else:
+         PgLOG.pglog(arg + ": Passed in without leading Option", PgLOG.LGWNEX)
+
+   if not scnames:
+      print("Usage: pgseq [-h HostName] [-d DatabaseName] -s SchemaNames  \\")
+      print("             [-t TableNames] [-u UserName] [-p Password]")
+      print("   -h: PostgreSQL Database server hostname, default to " + SINFO['ht'])
+      print("   -d: PostgreSQL Database name, default to " + SINFO['db'])
+      print("   -s: Reset sequences for the provided Schemas, at least provide one")
+      print("   -t: Table names to reset sequences, default to all tables in the schema")
+      print("   -u: Provide database login username if other than the schema name")
+      print("   -p: Optional database login password")
+      sys.exit(0)
+
+   PgLOG.cmdlog("pgseq {}".format(' '.join(argv)))
+   for scname in scnames:
+      reset_table_sequence(scname, tbnames)
+   PgLOG.cmdlog()
+   sys.exit(0)
+
+def reset_table_sequence(scname, tbnames):
+
+   us = SINFO['us'] if SINFO['us'] else scname
+   pw = SINFO['pw'] if SINFO['pw'] else us
+   PgDBI.default_scinfo(SINFO['db'], scname, SINFO['ht'], us, pw)
+   scnd = "table_catalog = '{}' AND table_schema = '{}' AND table_type = 'BASE TABLE'".format(SINFO['db'], scname)
+   if tbnames:
+      cnd = " AND table_name "
+      tcnt = len(tbnames)
+      if tcnt == 1:
+         cnd += "= '{}'".format(tbnames[0])
+      else:
+         cnd += "IN ('{}'".format(tbnames[0])
+         for i in range(1, tcnt):
+            cnd += ", '{}'".format(tbnames[i])
+         cnd += ')'
+   else:
+      cnd = ''
+   pgrecs = PgDBI.pgmget('information_schema.tables', 'table_name', scnd + cnd)
+   tbnames = pgrecs['table_name'] if pgrecs else []
+   tcnt = len(tbnames)
+   if tcnt == 0:
+      PgLOG.pglog("{}.{}: No Table Name found".format(SINFO['db'], scname), PgLOG.LOGWRN)
+      return
+
+   scnt = 0
+   scnd = "sequence_catalog = '{}' AND sequence_schema = '{}' AND sequence_name LIKE ".format(SINFO['db'], scname)
+   qscname = PgDBI.pgname(scname)
+   for tbname in tbnames:
+      cnd = "'{}_%_seq'".format(tbname)
+      pgrecs = PgDBI.pgmget('information_schema.sequences', 'sequence_name', scnd + cnd)
+      if not pgrecs: continue
+      qscname = PgDBI.pgname(scname)
+      qtname = '{}.{}'.format(qscname, PgDBI.pgname(tbname))
+      for sname in pgrecs['sequence_name']:
+         fname = sname[len(tbname)+1:-4]
+         fcnd = "table_name = '{}' AND table_schema = '{}' AND column_name = '{}'".format(tbname, scname, fname)
+         if not PgDBI.pgget('information_schema.columns', '', fcnd): continue
+         pgrec = PgDBI.pgget(qtname, "MAX({}) mval".format(PgDBI.pgname(fname)))
+         mval = pgrec['mval'] if pgrec and pgrec['mval'] else 1
+         qsname = '{}.{}'.format(qscname, PgDBI.pgname(sname))
+         qstr = "setval('{}', {})".format(qsname, mval)
+         pgrec = PgDBI.pgget('', qstr)
+         if pgrec:
+            PgLOG.pglog("{}.{}: Sequence Value set to {}".format(scname, sname, pgrec['setval']), PgLOG.LOGWRN)
+            scnt += 1
+
+   if scnt > 1 and tcnt > 1:
+      PgLOG.pglog("{}.{}: {}/{} Sequences/Tables Reset Values".format(SINFO['db'], scname, scnt, tcnt), PgLOG.LOGWRN)
+
+#
+# call main() to start program
+#
+if __name__ == "__main__": main()

rda_python_dbms-1.0.1.dist-info/METADATA

@@ -0,0 +1,17 @@
+Metadata-Version: 2.4
+Name: rda_python_dbms
+Version: 1.0.1
+Summary: RDA Python Package to manage PostreSQL databases
+Author-email: Zaihua Ji <zji@ucar.edu>
+Project-URL: Homepage, https://github.com/NCAR/rda-python-dbms
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Development Status :: 5 - Production/Stable
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: rda_python_common
+Dynamic: license-file
+
+RDA python Package to manage RDA PostgreSQL databases.

rda_python_dbms-1.0.1.dist-info/RECORD

@@ -0,0 +1,15 @@
+rda_python_dbms/MyDBI.py,sha256=VXtzR6zYojez2sPJreGlHKHgEJfF-V359T2mhSGJKHk,64376
+rda_python_dbms/PgDDLLib.py,sha256=DzBocr7c4c482zB8jxw0lughXOj327OL_TTx6ZvmjXA,40814
+rda_python_dbms/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+rda_python_dbms/pgbackup.py,sha256=ICYmuFCjlI_ICRaL33ZUGVzEE1CyEfC5xwhq3t9vJH0,3647
+rda_python_dbms/pgchksum.py,sha256=NlIW4N45nRYj16qF00lndiw5wyIB2YAFi5NfORRbuPk,19044
+rda_python_dbms/pgcmptables.py,sha256=y5Tx45BguJ4IVw0thgZmg9D9B2gcYMdem3shNugtAcA,47126
+rda_python_dbms/pgddl.py,sha256=0obj6Lsc0DVKSIvaP1n9d9TCMYb10f_Hoyc3Gg4jYnI,5441
+rda_python_dbms/pgddl.usg,sha256=7ly6YbusvyIy9yQn9reGIUtZQ5Wiehp9-DKDgfjqSBE,2267
+rda_python_dbms/pgseq.py,sha256=OkGhdYB03smGMo9fecdS3dZgR2jOm-ZODM9_1QjAlWE,4736
+rda_python_dbms-1.0.1.dist-info/licenses/LICENSE,sha256=Uky6fiiVw7cakFVRtVpzg-mvqOQ-WcA_H8lHqhPRg58,1101
+rda_python_dbms-1.0.1.dist-info/METADATA,sha256=CxpCsnO_spfFnD6vFY8aqAU5Xq8dB0q_N5w95PoPnpc,617
+rda_python_dbms-1.0.1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+rda_python_dbms-1.0.1.dist-info/entry_points.txt,sha256=e-Mc9poQZhN4C82G03V5Vyoh56jpz75-bpWkb7yG9l4,217
+rda_python_dbms-1.0.1.dist-info/top_level.txt,sha256=4T1ShNUMyiqmwDIuDL7pDOm_S-6zWWW3Vy6OdAzYA4c,16
+rda_python_dbms-1.0.1.dist-info/RECORD,,

rda_python_dbms-1.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (78.1.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

rda_python_dbms-1.0.1.dist-info/entry_points.txt

@@ -0,0 +1,6 @@
+[console_scripts]
+pgbackup = rda_python_dbms.pgbackup:main
+pgchksum = rda_python_dbms.pgchksum:main
+pgcmptables = rda_python_dbms.pgcmptables:main
+pgddl = rda_python_dbms.pgddl:main
+pgseq = rda_python_dbms.pgseq:main

rda_python_dbms-1.0.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 NSF National Center for Atmospheric Research
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

rda_python_dbms-1.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+rda_python_dbms