shareddata 2.1.4__tar.gz → 2.2.0__tar.gz

{shareddata-2.1.4 → shareddata-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: shareddata
-Version: 2.1.4
+Version: 2.2.0
 Summary: Shared Memory Database with S3 repository
 Home-page: https://github.com/jcarlitooliveira/SharedData
 Author: Jose Carlito de Oliveira Filho

{shareddata-2.1.4 → shareddata-2.2.0}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = shareddata
-version = 2.1.4
+version = 2.2.0
 author = Jose Carlito de Oliveira Filho
 author_email = jcarlitooliveira@gmail.com
 description = Shared Memory Database with S3 repository

shareddata-2.1.4/src/SharedData/SharedDataFrame.py → shareddata-2.2.0/src/SharedData/DataFrame.py

@@ -14,8 +14,8 @@ from datetime import datetime, timedelta
 
 
 from SharedData.Logger import Logger
-from SharedData.SharedDataAWSS3 import S3Download,S3Upload,UpdateModTime
-from SharedData.SharedDataRealTime import SharedDataRealTime
+from SharedData.AWSS3 import S3Download,S3Upload,UpdateModTime
+from SharedData.RealTime import SharedDataRealTime
 
 
 class SharedDataFrame:

{shareddata-2.1.4 → shareddata-2.2.0}/src/SharedData/Defaults.py

@@ -15,7 +15,7 @@ else:
         raise Exception('PYTHONHASHSEED must be set to 0 in the environment!')
 
 if not 'DATABASE_FOLDER' in os.environ:    
-    os.environ['DATABASE_FOLDER'] = os.path.expanduser("~")+'\DB' 
+    os.environ['DATABASE_FOLDER'] = os.path.expanduser("~")+'/db'
 
 if not 'S3_BUCKET' in os.environ:    
     os.environ['S3_BUCKET'] = 's3://deepportfolio'

shareddata-2.1.4/src/SharedData/SharedDataFeeder.py → shareddata-2.2.0/src/SharedData/Feeder.py

@@ -1,5 +1,5 @@
-from SharedData.SharedDataPeriod import SharedDataPeriod
-from SharedData.SharedDataTable import SharedDataTable
+from SharedData.Period import SharedDataPeriod
+from SharedData.Table import SharedDataTable
 
 class SharedDataFeeder():
 
@@ -12,7 +12,7 @@ class SharedDataFeeder():
         
         # DATA DICTIONARY
         # data[period][tag]
-        self.data = {}     
+        self.data = {}
     
     def __setitem__(self, dataset, value):
         if not dataset in self.data.keys():
@@ -24,7 +24,7 @@ class SharedDataFeeder():
         return self.data[dataset]
                 
     def __getitem__(self, dataset):
-        if not dataset in self.data.keys():            
+        if not dataset in self.data.keys():
             if (dataset in SharedDataFeeder.dense_datasets.keys()):                
                 period = dataset
                 self.data[period] = SharedDataPeriod(self, period)
@@ -34,14 +34,4 @@ class SharedDataFeeder():
         if (dataset in SharedDataFeeder.dense_datasets.keys()):
             return self.data[dataset]
         else:
-            return self.data[dataset].records
-    
-    def create_table(self,dataset,names,formats,size,overwrite=False):
-        self.data[dataset] = SharedDataTable(\
-            self,dataset,names=names,formats=formats,size=size,\
-                overwrite=overwrite)
-        return self.data[dataset].records
-    
-    def load_table(self,dataset,size=None):
-        self.data[dataset] = SharedDataTable(self,dataset,size=size)
-        return self.data[dataset].records
+            return self.data[dataset].records

{shareddata-2.1.4 → shareddata-2.2.0}/src/SharedData/Logger.py

@@ -8,19 +8,18 @@ import pandas as pd
 from pythonjsonlogger.jsonlogger import JsonFormatter
 import boto3
 
-from importlib.metadata import version
-
-import SharedData.Defaults as Defaults 
-from SharedData.SharedDataAWSKinesis import KinesisLogStreamHandler
 
+from SharedData.AWSKinesis import KinesisLogStreamHandler
 
 class Logger:
 
     log = None
-    user = None
+    user = 'guest'
+    source = 'unknown'
 
-    def __init__(self, source, user=None):
-        if Logger.log is None:                                
+    @staticmethod
+    def connect(source,user=None):
+        if Logger.log is None:
             if 'SOURCE_FOLDER' in os.environ:
                 try:
                     commompath = os.path.commonpath([source,os.environ['SOURCE_FOLDER']])
@@ -42,13 +41,10 @@ class Logger:
             source = source.lstrip('src').lstrip('\\src')
             source = source.lstrip('\\').lstrip('/')        
             source = source.replace('.py','')
-            self.source = source
+            Logger.source = source
             
             if not user is None:                
                 Logger.user = user
-            else:
-                user = 'guest'
-                Logger.user = 'guest'
                         
             path = Path(os.environ['DATABASE_FOLDER'])
             path = path / 'Logs'
@@ -68,8 +64,8 @@ class Logger:
                 loglevel = logging.INFO
 
             # Create Logger
-            self.log = logging.getLogger(source)
-            self.log.setLevel(logging.DEBUG)    
+            Logger.log = logging.getLogger(source)
+            Logger.log.setLevel(logging.DEBUG)    
             formatter = logging.Formatter(os.environ['USERNAME']+'@'+os.environ['COMPUTERNAME'] + 
                 ';%(asctime)s;%(name)s;%(levelname)s;%(message)s',\
                 datefmt='%Y-%m-%dT%H:%M:%S%z')
@@ -77,12 +73,12 @@ class Logger:
             handler = logging.StreamHandler()
             handler.setLevel(loglevel)    
             handler.setFormatter(formatter)
-            self.log.addHandler(handler)
-            #log file        
-            fhandler = logging.FileHandler(os.environ['LOG_PATH'], mode='a')
-            fhandler.setLevel(loglevel)    
-            fhandler.setFormatter(formatter)
-            self.log.addHandler(fhandler)
+            Logger.log.addHandler(handler)
+            # #log file        
+            # fhandler = logging.FileHandler(os.environ['LOG_PATH'], mode='a')
+            # fhandler.setLevel(loglevel)    
+            # fhandler.setFormatter(formatter)
+            # Logger.log.addHandler(fhandler)
             #log to aws kinesis
             kinesishandler = KinesisLogStreamHandler(user=Logger.user)
             kinesishandler.setLevel(logging.DEBUG)
@@ -90,15 +86,10 @@ class Logger:
                 ';%(asctime)s;%(name)s;%(levelname)s;%(message)s',\
                 datefmt='%Y-%m-%dT%H:%M:%S%z')
             kinesishandler.setFormatter(jsonformatter)
-            self.log.addHandler(kinesishandler)
-            #assign static variable log to be used by modules
-            Logger.log = self.log
-            try:
-                Logger.log.debug('Logger initialized shareddata user %s version %s!' % \
-                    (Logger.user,version('SharedData')))
-            except:
-                Logger.log.debug('Logger initialized shareddata!')
+            Logger.log.addHandler(kinesishandler)            
+            
         
+    @staticmethod
     def readLogs(self):    
         logsdir = Path(os.environ['LOG_PATH']).parents[0]
         lenlogsdir = len(str(logsdir))

{shareddata-2.1.4 → shareddata-2.2.0}/src/SharedData/LoggerConsumerProcess.py

@@ -3,7 +3,7 @@ import numpy as np
 
 from SharedData.Logger import Logger
 logger = Logger(__file__,user='worker')
-from SharedData.SharedDataAWSKinesis import KinesisLogStreamConsumer
+from SharedData.AWSKinesis import KinesisLogStreamConsumer
 
 SLEEP_TIME = 2
 

{shareddata-2.1.4 → shareddata-2.2.0}/src/SharedData/Metadata.py

@@ -9,7 +9,7 @@ from datetime import datetime, timedelta
 import gzip,io,shutil,hashlib
 
 from SharedData.Logger import Logger
-from SharedData.SharedDataAWSS3 import S3Upload, S3ListFolder, S3Download, UpdateModTime
+from SharedData.AWSS3 import S3Upload, S3ListFolder, S3Download, UpdateModTime
 
 class Metadata():
     

shareddata-2.1.4/src/SharedData/SharedDataPeriod.py → shareddata-2.2.0/src/SharedData/Period.py

@@ -1,4 +1,4 @@
-import os,io,hashlib,gzip,shutil
+import os,io,hashlib,gzip,shutil,time
 import pandas as pd
 import numpy as np
 from datetime import datetime, timedelta
@@ -6,11 +6,10 @@ from pathlib import Path
 from pandas.tseries.offsets import BDay
 from threading import Thread
 
-
 from SharedData.Logger import Logger
-from SharedData.SharedDataFrame import SharedDataFrame
-from SharedData.SharedDataTimeSeries import SharedDataTimeSeries
-from SharedData.SharedDataAWSS3 import S3Upload,S3Download,UpdateModTime
+from SharedData.DataFrame import SharedDataFrame
+from SharedData.TimeSeries import SharedDataTimeSeries
+from SharedData.AWSS3 import S3Upload,S3Download,UpdateModTime
 
 class SharedDataPeriod:
 
@@ -41,16 +40,6 @@ class SharedDataPeriod:
         self.getContinousTimeIndex(self.default_startDate)
         self.loaded = False
 
-        
-    def load(self):
-        # read if not loaded
-        shdatalist = self.sharedData.list()   
-        path, shm_name = self.get_path()
-        idx = [shm_name in str(s) for s in shdatalist.index]
-        if not np.any(idx):
-            self.read()
-
-
     def __setitem__(self, tag, df):
         if not tag in self.tags.keys():
             if isinstance(tag, pd.Timestamp):
@@ -63,9 +52,9 @@ class SharedDataPeriod:
             icol = df.columns.intersection(data.columns)
             data.loc[iidx, icol] = df.loc[iidx, icol].copy()
 
-    def __getitem__(self, tag):     
+    def __getitem__(self, tag):
         if not self.loaded:
-            self.load()   
+            self.load()
             self.loaded=True
         if not tag in self.tags.keys():
             if isinstance(tag, pd.Timestamp):
@@ -125,30 +114,6 @@ class SharedDataPeriod:
             self.ctimeidx[startDate][periods.values] = np.arange(len(periods))        
         return self.ctimeidx[startDate]
 
-    def get(self, tag):
-        if isinstance(tag, pd.Timestamp):
-            df = SharedDataFrame(self, tag, malloc=False)
-            return df.data
-        else:  
-            return pd.DataFrame([]) 
-    
-    def exists(self, tag):
-        if isinstance(tag, pd.Timestamp):            
-            path = Path(os.environ['DATABASE_FOLDER'])
-            path = path / self.sharedData.user
-            path = path / self.sharedData.database
-            path = path / self.sharedDataFeeder.feeder
-            path = path / self.period       
-            fpath = path / (tag.strftime('%Y%m%d%H%M')+'.npy')            
-            return fpath.is_file()
-        else:
-            return False 
- 
-    def create_timeseries(self,tag,startDate,columns,overwrite=False):
-        self.tags[tag] = SharedDataTimeSeries(\
-            self,tag,startDate=startDate,columns=columns,overwrite=overwrite)
-        return self.tags[tag].data
-
     def get_path(self):        
         shm_name = self.sharedData.user + '/' + self.sharedData.database + '/' \
             + self.sharedDataFeeder.feeder + '/' + self.period
@@ -166,7 +131,31 @@ class SharedDataPeriod:
                 os.makedirs(path.parent)
         return path , shm_name
 
+    #CREATE
+    def create_timeseries(self,tag,startDate,columns,overwrite=False):
+        self.tags[tag] = SharedDataTimeSeries(\
+            self,tag,startDate=startDate,columns=columns,overwrite=overwrite)
+        return self.tags[tag].data
+
+    # READ
+    def load(self):
+        # read if not loaded
+        shdatalist = self.sharedData.list()   
+        path, shm_name = self.get_path()
+        idx = [shm_name in str(s) for s in shdatalist.index]
+        if not np.any(idx):
+            self.read()
+        else:
+            #map
+            self.map(shm_name,shdatalist.index[idx])
+
+    def map(self,shm_name,shm_name_list):
+        for shm in shm_name_list:
+            tag = shm.replace(shm_name,'')[1:]
+            self.tags[tag] = SharedDataTimeSeries(self, tag = tag)            
+
     def read(self):
+        tini = time.time()
         datasize = 1
         path, shm_name= self.get_path()
         headpath = str(path)+'_head.bin'
@@ -209,20 +198,27 @@ class SharedDataPeriod:
                 tail_io = open(str(tailpath),'rb')
 
         if not head_io is None:
-            self.read_io(head_io,headpath)
+            datasize += self.read_io(head_io,headpath)
 
         if not tail_io is None:
-            self.read_io(tail_io,tailpath)
+            datasize += self.read_io(tail_io,tailpath)
+
+        te = time.time()-tini+0.000001   
+        datasize = datasize/(1024*1024)
+        Logger.log.debug('read %s/%s %.2fMB in %.2fs %.2fMBps ' % \
+            (self.sharedDataFeeder,self.period,datasize,te,datasize/te))
 
     def read_io(self,io_obj,path):
+        datasize = 0        
         #read
         io_obj.seek(0)
         io_data = io_obj.read()
+        datasize = len(io_data)
         _m = hashlib.md5(io_data[:-24]).digest()
         m = io_data[-16:]
         if (m!=_m):
-            Logger.log.error('Timeseries read_head() file %s corrupted!' % (path))
-            raise Exception('Timeseries read_head() %s corrupted!' % (path))
+            Logger.log.error('Timeseries file %s corrupted!' % (path))
+            raise Exception('Timeseries file %s corrupted!' % (path))
         io_obj.seek(0)
         separator = np.frombuffer(io_obj.read(8),dtype=np.int64)[0]        
         while (separator==1):
@@ -249,6 +245,9 @@ class SharedDataPeriod:
             separator = np.frombuffer(io_obj.read(8),dtype=np.int64)[0]
         io_obj.close()
 
+        return datasize
+
+    # WRITE
     def write(self,startDate=None):
         path , shm_name= self.get_path()                
         mtime = datetime.now().timestamp()

shareddata-2.1.4/src/SharedData/SharedDataRealTime.py → shareddata-2.2.0/src/SharedData/RealTime.py

@@ -4,7 +4,7 @@ import pandas as pd
 import numpy as np
 
 from SharedData.Logger import Logger
-from SharedData.SharedDataAWSKinesis import KinesisStreamProducer
+from SharedData.AWSKinesis import KinesisStreamProducer
 
 class SharedDataRealTime:
 

shareddata-2.1.4/src/SharedData/SharedDataRealTimeProcess.py → shareddata-2.2.0/src/SharedData/RealTimeProcess.py

@@ -6,7 +6,7 @@ from pathlib import Path
 from SharedData.Logger import Logger
 logger = Logger(__file__)
 from SharedData.SharedData import SharedData
-from SharedData.SharedDataAWSKinesis import KinesisStreamConsumer
+from SharedData.AWSKinesis import KinesisStreamConsumer
 
 
 try:

shareddata-2.2.0/src/SharedData/SharedData.py

@@ -0,0 +1,197 @@
+import os, psutil
+import pandas as pd
+from multiprocessing import shared_memory
+from pathlib import Path
+from importlib.metadata import version
+
+import SharedData.Defaults as Defaults 
+from SharedData.Logger import Logger
+from SharedData.Table import Table
+from SharedData.TimeseriesContainer import TimeseriesContainer
+from SharedData.TimeSeries import TimeSeries
+from SharedData.DataFrame import SharedDataFrame
+from SharedData.RealTime import SharedDataRealTime
+from SharedData.Utils import remove_shm_from_resource_tracker
+
+class SharedData:    
+    
+    def __init__(self,source,user='guest'):
+        self.source = source
+        self.user = user
+
+        # DATA DICTIONARY        
+        self.data = {}
+        
+        # LOGIN VARIABLES
+        self.islogged = False
+        self.source=source
+        self.user=user
+        self.mode='rw'
+
+        # S3 VARIABLES
+        self.s3read = True
+        self.s3write = True
+
+        # save files locally
+        self.save_local = (os.environ['SAVE_LOCAL'] == 'True')
+
+        self.databases = {
+            'MarketData' : ['date','symbol'],
+            'Relationships' : ['date','symbol1','symbol2'],
+            'Portfolios' : ['date','portfolio'],
+            'Signals' : ['date','portfolio','symbol'],
+            'Risk' : ['date','portfolio','symbol'],
+            'Positions' : ['date','portfolio','symbol'],
+            'Orders' : ['date','portfolio','symbol','clordid'],
+            'Trades' : ['date','portfolio','symbol','tradeid']
+        }
+        
+        Logger.connect(self.source,self.user)
+
+        if (os.name == 'posix'):
+            remove_shm_from_resource_tracker()        
+    
+        if not self.islogged:
+            self.islogged = True
+            try:
+                Logger.log.debug('%s@%s CONNECTED!\nSharedData version %s, user %s' %
+                    (os.environ['USERNAME'], os.environ['COMPUTERNAME'],version('SharedData'), self.user))
+            except:
+                Logger.log.debug('%s:%s CONNECTED!\nSharedData user %s!' %
+                    (os.environ['USERNAME'], os.environ['COMPUTERNAME'],self.user))
+            
+    ###############################################
+    ############# DATA CONTAINERS #################
+    ###############################################
+
+    ############# TABLE #################
+    def table(self,database,period,source,tablename,\
+        names=None,formats=None,size=None,value=None,user='master'):
+
+        path = user+'/'+database+'/'+period+'/'+source+'/table/'+tablename
+        if not path in self.data.keys():
+            self.data[path] = Table(self,database,period,source,\
+                tablename,records=value,names=names,formats=formats,size=size,user=user)
+        return self.data[path].records
+    
+    ############# TIMESERIES #################
+    def timeseries(self,database,period,source,tag=None,\
+        startDate=None,columns=None,value=None,user='master'):
+        
+        path = user+'/'+database+'/'+period+'/'+source+'/timeseries'
+        if not path in self.data.keys():
+            self.data[path] = TimeseriesContainer(self, database, period, source,user=user)
+        if tag is None:
+            return self.data[path]
+        
+        if not tag in self.data[path].tags.keys():
+            if (startDate is None) & (columns is None) & (value is None):
+                self.data[path].load()
+                if not tag in self.data[path].tags.keys():
+                    Logger.log.error('Tag %s/%s doesnt exist' % (path,tag))
+                    return None
+            else:
+                self.data[path].tags[tag] = TimeSeries(self, self.data[path],\
+                    database, period, source, tag,\
+                    value=value,startDate=startDate,columns=columns,user=user)
+                            
+        return self.data[path].tags[tag].data
+            
+    ############# DATAFRAME #################
+    def dataframe(self,database,period,source,\
+        date=None,value=None,user='master'):
+        pass
+
+    ###############################################
+    ######### SHARED MEMORY MANAGEMENT ############
+    ###############################################    
+    @staticmethod
+    def malloc(shm_name,create=False,size=None,overwrite=False):
+        ismalloc = False
+        shm = None        
+        try:
+            shm = shared_memory.SharedMemory(\
+                name = shm_name,create=False)
+            ismalloc = True
+        except:
+            pass
+
+        if (not ismalloc) & (create) & (not size is None):
+            shm = shared_memory.SharedMemory(\
+                name=shm_name,create=True,size=size)
+            ismalloc = True
+        
+        elif (create) & (size is None):
+            raise Exception('SharedData malloc must have a size when create=True')
+        
+        elif (os.name=='posix')\
+            & (ismalloc) & (create) & (overwrite) & (not size is None):
+            SharedData.free(shm_name)
+            shm = shared_memory.SharedMemory(\
+                name=shm_name,create=True,size=size)
+            ismalloc = True            
+        
+        # register process id access to memory
+        if ismalloc:            
+            fpath = Path(os.environ['DATABASE_FOLDER'])
+            fpath = fpath/('shm/'+shm_name.replace('\\','/')+'.csv')
+            os.makedirs(fpath.parent,exist_ok=True)
+            pid = os.getpid()
+            f = open(fpath, "a+")
+            f.write(str(pid)+',')
+            f.flush()
+            f.close()
+
+        return [shm, ismalloc]
+    
+    @staticmethod
+    def free(shm_name):
+        if os.name=='posix':
+            try:
+                shm = shared_memory.SharedMemory(\
+                    name = shm_name,create=False)
+                shm.close()
+                shm.unlink()
+                fpath = Path(os.environ['DATABASE_FOLDER'])
+                fpath = fpath/('shm/'+shm_name.replace('\\','/')+'.csv')
+                if fpath.is_file():
+                    os.remove(fpath)
+            except:
+                pass
+
+    @staticmethod
+    def list():
+        folder = Path(os.environ['DATABASE_FOLDER'])/'shm'
+        shm_names = pd.DataFrame()
+        for root, _, filenames in os.walk(folder):
+            for filename in filenames:
+                if filename.endswith('.csv'):
+                    fpath = os.path.join(root, filename)
+                    shm_name = fpath.removeprefix(str(folder))[1:]
+                    shm_name = shm_name.removesuffix('.csv')
+                    shm_name = shm_name.replace('/','\\')
+                    try:
+                        shm = shared_memory.SharedMemory(\
+                            name = shm_name,create=False)
+                        shm_names.loc[shm_name,'size'] = shm.size
+                        shm.close()                
+                    except:
+                        try:                    
+                            if fpath.is_file():
+                                os.remove(fpath)                    
+                        except:
+                            pass
+        shm_names = shm_names.sort_index()
+        return shm_names
+    
+    @staticmethod
+    def freeall():
+        shm_names = SharedData.list()
+        for shm_name in shm_names.index:
+            SharedData.free(shm_name)
+    
+    ###############################################
+    ############# REAL TIME MANAGEMENT ############
+    ###############################################
+    def subscribe(database,source,period):
+        SharedDataRealTime.subscribe(database,source,period)