satnogs-webscraper 1.4.2__tar.gz → 1.4.4__tar.gz

{satnogs_webscraper-1.4.2 → satnogs_webscraper-1.4.4}/PKG-INFO

@@ -1,19 +1,24 @@
 Metadata-Version: 2.1
 Name: satnogs_webscraper
-Version: 1.4.2
+Version: 1.4.4
 Summary: Scrape SATNOGS data into a pandas dataframe
-Home-page: UNKNOWN
 Author: Ada L
 Author-email: the.nostra.tymus@gmail.com
 License: MIT
 Keywords: satellite satnogs
-Platform: UNKNOWN
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
 Description-Content-Type: text/markdown
 License-File: LICENSE.rst
+Requires-Dist: pillow
+Requires-Dist: numpy
+Requires-Dist: requests
+Requires-Dist: bs4
+Requires-Dist: html5lib
+Requires-Dist: pandas
+Requires-Dist: dask[complete]
+Requires-Dist: pytest
 
 A webscraper to pull multiple satnogs observations into a single pandas dataframe
-

{satnogs_webscraper-1.4.2 → satnogs_webscraper-1.4.4}/satnogs_webscraper/constants.py

@@ -6,18 +6,18 @@ web_address = "https://network.satnogs.org/"
 
 observation_template = {
     'Observation_id': None,
+    'Timeframe': None,
     'Satellite': None,
     'Station': None,
     'Status': None,
     'Status_Message': None,
-    'Transmitter': None,
     'Frequency': None,
     'Mode': None,
     'Metadata': None,
     'Downloads': None,
     'Waterfall_Status': None,
     'Polar_Plot': None,
-    'demods':None
+    'demods': None
 }
 
 directories = dict()

{satnogs_webscraper-1.4.2 → satnogs_webscraper-1.4.4}/satnogs_webscraper/image_utils.py

@@ -73,12 +73,18 @@ def crop_and_save_psd(input_image, delete_original=True, greyscale=True, resize=
             im_cropped = im_cropped.resize(resize_dimen, Image.Resampling.LANCZOS)
 
     # Convert to greyscale and save as unit8 bytes to disk, using the original file name, minus the file extension
-    numpy_im = np.array(im_cropped)
 
-    # store the shape and write to a file
-    shape = numpy_im.shape
-    new_file_name = input_image[:-4]
-    numpy_im.tofile(new_file_name)
+    new_file_name = "".join(input_image.split(".")[0:-1])
+    new_file_name += ".bin" if greyscale else ".png"
+
+    if greyscale:
+        numpy_im = np.array(im_cropped)
+        shape = numpy_im.shape
+        numpy_im.tofile(new_file_name)
+    else:
+        numpy_im = np.array(im_cropped)
+        shape = numpy_im.shape
+        im_cropped.save(new_file_name)
 
     # remove the original, larger image
     if delete_original:

satnogs_webscraper-1.4.4/satnogs_webscraper/observation_dataset.py

@@ -0,0 +1,115 @@
+import datetime
+import json
+import os
+import pandas as pd
+import re
+import satnogs_webscraper.constants as cnst
+import warnings
+
+
+def save_dataset(observation_list, save_name):
+    df = get_dataset(observation_list)
+    df.to_csv(save_name, index=False)
+
+
+def get_dataset(observation_list):
+    observations = []
+    for observation in observation_list:
+        file_name = os.path.join(cnst.directories['observations'], f'{observation}.json')
+        with open(file_name, "r") as file_in:
+            observations.append(json.load(file_in))
+
+    df = pd.DataFrame(observations)
+    return df
+
+
+def get_datasets(observation_list, kaitai_interface=None):
+    common_key = 0
+
+    observations = []
+    demod_dictionary = dict()
+    demods = []
+
+    for observation in observation_list:
+        file_name = os.path.join(cnst.directories['observations'], f'{observation}.json')
+        with open(file_name, "r") as file_in:
+            observation_dictionary = json.load(file_in)
+            observation_dictionary['meta_key'] = common_key if len(observation_dictionary['demods']) > 0 else -1
+
+        for demod in observation_dictionary['demods']:
+            with open(demod['location'], 'rb') as file_in:
+                demod_data = file_in.read()
+                demods.append({
+                    'timestamp': get_demod_time(demod['original_name']),
+                    'bytes': demod_data,
+                    'meta_key': common_key
+                })
+                if kaitai_interface is not None:
+                    from kaitaistruct import KaitaiStream, BytesIO
+                    if 'kaitai' not in observation_dictionary.keys():
+                        observation_dictionary['kaitai'] = []
+                    try:
+                        observation_dictionary['kaitai'].append(kaitai_interface(KaitaiStream(BytesIO(demod_data))))
+                    except Exception as e:
+                        warnings.warn(f"Error parsing demod. Error:{e}")
+                        pass
+        observations.append(observation_dictionary)
+        common_key += 1
+
+    meta_df = pd.DataFrame(observations)
+    meta_df.set_index(['Observation_id'], inplace=True)
+    meta_df.sort_index(inplace=True)
+
+    for demod in demods:
+        length_key = len(demod['bytes'])
+
+        if length_key not in demod_dictionary.keys():
+            demod_dictionary[length_key] = dict()
+            demod_dictionary[length_key]['demods'] = []
+            demod_dictionary[length_key]['demod_length'] = length_key
+            demod_dictionary[length_key]['num_demods'] = 0
+
+        for k, v in enumerate(demod['bytes']):
+            demod[k] = v
+
+        del demod['bytes']
+
+        demod_dictionary[length_key]['demods'].append(demod)
+        demod_dictionary[length_key]['num_demods'] += 1
+
+    for key in demod_dictionary.keys():
+        df = pd.DataFrame(demod_dictionary[key]['demods'])
+        df.set_index(['timestamp'], inplace=True)
+        df.sort_index(inplace=True)
+        demod_dictionary[key]['dataframe'] = df
+        del demod_dictionary[key]['demods']
+
+    if len(demod_dictionary.keys()) != 0:
+        demod_df = pd.DataFrame([demod_dictionary[key] for key in demod_dictionary.keys()])
+        demod_df.set_index(['demod_length'], inplace=True)
+        demod_df.sort_values(['num_demods'], ascending=False, inplace=True)
+        return meta_df, demod_df
+    else:
+        return meta_df, None
+
+
+def get_demod_time(demod_url):
+
+    pattern = r"\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}(?:_\d+)?"
+    match = re.search(pattern, demod_url)
+    time_string = match.group(0)
+
+    if time_string.find("_") == -1:
+        time_parsed = datetime.datetime.strptime(time_string, "%Y-%m-%dT%H-%M-%S")
+    else:
+        time_stamp_part = time_string.split("_")[0]
+        extra_part = int(time_string.split("_")[1])
+        time_parsed = datetime.datetime.strptime(time_stamp_part, "%Y-%m-%dT%H-%M-%S")
+        time_parsed = time_parsed + datetime.timedelta(milliseconds=extra_part)
+    return time_parsed
+
+
+def find_largest_number(string):
+    numbers = re.findall(r'\d+', string)
+    numbers = list(map(int, numbers))
+    return max(numbers) if numbers else 0

{satnogs_webscraper-1.4.2 → satnogs_webscraper-1.4.4}/satnogs_webscraper/observation_scraper.py

@@ -17,7 +17,7 @@ import satnogs_webscraper.progress_utils as pu
 
 class ObservationScraper:
     def __init__(self, fetch_waterfalls=True, fetch_logging=True, prints=True, check_disk=True, cpus=1,
-                 grey_scale=True):
+                 grey_scale=True, delete_original_waterfall=True):
         """
         Scrapes the webpages for satellite observations. Waterfall fetches are set to false by default due to the
         very large file sizes.
@@ -34,7 +34,7 @@ class ObservationScraper:
         self.log_file_loc = cnst.files["log_file"]
         self.waterfall_path = cnst.directories['waterfalls']
         self.demod_path = cnst.directories['demods']
-
+        self.delete_original_waterfall = delete_original_waterfall
         self.prints = prints
         self.check_disk = check_disk
         cnst.verify_directories()
@@ -49,7 +49,7 @@ class ObservationScraper:
         """
         urls = [f'{cnst.web_address}{cnst.observations}{observation}/' for observation in observations_list]
 
-        #self.progress_dict = pu.setup_progress_dict(items_total=len(urls), items_done=0)
+        # self.progress_dict = pu.setup_progress_dict(items_total=len(urls), items_done=0)
 
         # pool = Pool(self.cpus)
         # self.observations_list = pool.map(self.scrape_observation, urls)
@@ -90,6 +90,10 @@ class ObservationScraper:
             if waterfall_status is not None:
                 template['Waterfall_Status'] = waterfall_status.text.strip()
 
+            meta_data_element = observation_web_page.find('pre', id='json-metadata')
+            meta_data_json = meta_data_element['data-json']
+            template['Metadata'] = json.loads(meta_data_json)
+
             status = observation_web_page.select("#rating-status > span")
             if (status is not None) & (status[0] is not None):
                 template['Status'] = status[0].text.strip()
@@ -97,10 +101,12 @@ class ObservationScraper:
             template['Observation_id'] = observation
 
             template['demods'] = []
+            demod_tasks = []
 
             for data_a in observation_web_page.find_all("a", class_='data-link'):
-                template['demods'].append(self.fetch_demod(data_a))
+                demod_tasks.append(dask.delayed(self.fetch_demod)(data_a))
 
+            template['demods'] = dask.compute(*[demod_tasks])[0]
             with open(os.path.join(cnst.directories['observations'], f"{observation}.json"), 'w') as obs_out:
                 json.dump(template, obs_out)
 
@@ -133,7 +139,7 @@ class ObservationScraper:
             except:
                 return "Satellite", ""
 
-        if contents.find("Station") != -1:
+        if (contents.find("Station") != -1) and (contents.find("Owner") == -1):
             try:
                 second_element = tr.select_one('td:nth-child(2)')
                 second_element = second_element.find("a")
@@ -141,12 +147,14 @@ class ObservationScraper:
             except:
                 return "Station", ""
 
-        if contents.find("Transmitter") != -1:
+        if contents.find("Timeframe") != -1:
             try:
                 second_element = tr.select_one('td:nth-child(2)')
-                return "Transmitter", second_element.text.strip()
+                dates = second_element.find_all('span', class_='datetime-date')
+                times = second_element.find_all('span', class_='datetime-time')
+                return "Timeframe", [f"{date.text} {time.text}" for date, time in zip(dates, times)]
             except:
-                return "Transmitter", ""
+                return "Timeframe", []
 
         if contents.find("Frequency") != -1:
             try:
@@ -159,18 +167,11 @@ class ObservationScraper:
         if contents.find("Mode") != -1:
             try:
                 second_element = tr.select_one('td:nth-child(2)')
-                return "Mode", [span.text.strip() for span in second_element.select("span") if span is not None]
+                return "Mode", " ".join(
+                    [span.text.strip() for span in second_element.select("span") if span is not None])
             except:
                 return "Mode", ""
 
-        if contents.find("Metadata") != -1:
-            try:
-                second_element = tr.select_one('td:nth-child(2)')
-                element = second_element.find("pre")
-                return "Metadata", element.attrs['data-json']
-            except:
-                return "Metadata", ""
-
         if contents.find("Polar Plot") != -1:
             element = tr.select_one("svg")
             try:
@@ -213,12 +214,13 @@ class ObservationScraper:
         :return: The shape of the cropped image and name of the waterfall written to disk as a bytes object.
         """
         res = ru.get_request(url)
-        waterfall_name = os.path.abspath(self.waterfall_path + file_name)
+        waterfall_name = os.path.abspath(self.waterfall_path + file_name) +"_original"
 
         with open(waterfall_name, 'wb') as out:
             out.write(res.content)
 
-        cropped_shape, bytes_name = iu.crop_and_save_psd(waterfall_name, greyscale=self.grey_scale)
+        cropped_shape, bytes_name = iu.crop_and_save_psd(waterfall_name, greyscale=self.grey_scale,
+                                                         delete_original=self.delete_original_waterfall)
 
         return cropped_shape, bytes_name
 

{satnogs_webscraper-1.4.2 → satnogs_webscraper-1.4.4}/satnogs_webscraper/request_utils.py

@@ -7,7 +7,7 @@ import time
 import satnogs_webscraper.constants as cnst
 
 
-def get_request(url):
+def get_request(url, max_count=None):
     count = 1
     while True:
         try:
@@ -17,7 +17,7 @@ def get_request(url):
                 return res
             else:
                 sleep_amount = count ** 2
-                write_log(url,res.status_code, f"Timeout Count: {count}")
+                write_log(url, res.status_code, f"Timeout Count: {count}")
                 time.sleep(sleep_amount)
         except Exception as e:
             sleep_amount = count ** 2
@@ -26,6 +26,9 @@ def get_request(url):
             if count == 10:
                 return None
         count += 1
+        if max_count is not None:
+            if count >= max_count:
+                return None
 
 
 def write_log(url, code, comment=""):

{satnogs_webscraper-1.4.2 → satnogs_webscraper-1.4.4}/satnogs_webscraper/scraper.py

@@ -6,7 +6,7 @@ import os
 import satnogs_webscraper.constants as cnst
 import satnogs_webscraper.observation_scraper as obs
 import satnogs_webscraper.observation_list_scraper as ols
-from satnogs_webscraper.observation_dataset import get_dataset
+from satnogs_webscraper.observation_dataset import get_dataset, get_datasets
 
 
 class Results(Enum):
@@ -43,6 +43,8 @@ class Scraper:
     save_name: str = "scrape"
     cpus: int = 0
     grey_scale: bool = True
+    kaitai_interface: object = None
+    meta_df_only: bool = False
 
     def __post_init__(self):
         self.waterfall = Results(self.waterfall)
@@ -133,7 +135,11 @@ class Scraper:
         print("Scraping Observation Pages...")
         obs_scraper = obs.ObservationScraper(cpus=self.cpus, grey_scale=self.grey_scale)
         obs_scraper.multiprocess_scrape_observations(ids)
-        return get_dataset(ids)
+
+        if self.meta_df_only:
+            return get_dataset(ids)
+        else:
+            return get_datasets(ids, self.kaitai_interface)
 
 
 if __name__ == '__main__':

{satnogs_webscraper-1.4.2 → satnogs_webscraper-1.4.4}/satnogs_webscraper.egg-info/PKG-INFO

@@ -1,19 +1,24 @@
 Metadata-Version: 2.1
-Name: satnogs-webscraper
-Version: 1.4.2
+Name: satnogs_webscraper
+Version: 1.4.4
 Summary: Scrape SATNOGS data into a pandas dataframe
-Home-page: UNKNOWN
 Author: Ada L
 Author-email: the.nostra.tymus@gmail.com
 License: MIT
 Keywords: satellite satnogs
-Platform: UNKNOWN
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
 Description-Content-Type: text/markdown
 License-File: LICENSE.rst
+Requires-Dist: pillow
+Requires-Dist: numpy
+Requires-Dist: requests
+Requires-Dist: bs4
+Requires-Dist: html5lib
+Requires-Dist: pandas
+Requires-Dist: dask[complete]
+Requires-Dist: pytest
 
 A webscraper to pull multiple satnogs observations into a single pandas dataframe
-

{satnogs_webscraper-1.4.2 → satnogs_webscraper-1.4.4}/satnogs_webscraper.egg-info/SOURCES.txt

@@ -1,6 +1,5 @@
 LICENSE.rst
 README.md
-README.rst
 setup.cfg
 setup.py
 satnogs_webscraper/__init__.py
@@ -17,4 +16,11 @@ satnogs_webscraper.egg-info/PKG-INFO
 satnogs_webscraper.egg-info/SOURCES.txt
 satnogs_webscraper.egg-info/dependency_links.txt
 satnogs_webscraper.egg-info/requires.txt
-satnogs_webscraper.egg-info/top_level.txt
+satnogs_webscraper.egg-info/top_level.txt
+tests/test_constants.py
+tests/test_image_utils.py
+tests/test_observation_dataset.py
+tests/test_observation_list_scraper.py
+tests/test_observation_scraper.py
+tests/test_request_utils.py
+tests/test_scraper.py

{satnogs_webscraper-1.4.2 → satnogs_webscraper-1.4.4}/setup.cfg

@@ -1,5 +1,5 @@
 [metadata]
-description-file = README.md
+description_file = README.md
 license_files = LICENSE.rst
 
 [egg_info]

{satnogs_webscraper-1.4.2 → satnogs_webscraper-1.4.4}/setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup
 
-VERSION = '1.4.2'
+VERSION = '1.4.4'
 DESCRIPTION = 'Scrape SATNOGS data into a pandas dataframe'
 LONG_DESCRIPTION = 'A webscraper to pull multiple satnogs observations into a single pandas dataframe'
 

satnogs_webscraper-1.4.4/tests/test_constants.py

@@ -0,0 +1,16 @@
+import os
+import shutil
+import satnogs_webscraper.constants as cnst
+
+
+def test_verify_directories():
+    if os.path.exists(cnst.directories['data']):
+        shutil.rmtree(cnst.directories['data'])
+
+    cnst.verify_directories()
+
+    for key in cnst.directories.keys():
+        assert os.path.exists(cnst.directories[key]), f"Verify {key}:{cnst.directories[key]} exists"
+
+    if os.path.exists(cnst.directories['data']):
+        shutil.rmtree(cnst.directories['data'])

satnogs_webscraper-1.4.4/tests/test_image_utils.py

@@ -0,0 +1,69 @@
+import os
+import shutil
+from PIL import Image
+
+import satnogs_webscraper.image_utils as iu
+from satnogs_webscraper.observation_scraper import ObservationScraper
+import satnogs_webscraper.constants as cnst
+import pytest
+
+
+@pytest.fixture()
+def get_image_greyscale():
+    cnst.verify_directories()
+    scraper = ObservationScraper(grey_scale=False, delete_original_waterfall=False)
+    scrape = scraper.scrape_observation('https://network.satnogs.org/observations/5025420/')
+    original_name = scrape['Downloads']['waterfall_hash_name']+"_original"
+    yield Image.open(original_name).convert('L')
+    shutil.rmtree(cnst.directories['data'])
+
+
+
+@pytest.fixture()
+def get_image():
+    cnst.verify_directories()
+    scraper = ObservationScraper(grey_scale=False, delete_original_waterfall=False)
+    scrape = scraper.scrape_observation('https://network.satnogs.org/observations/5025420/')
+    original_name = scrape['Downloads']['waterfall_hash_name']+"_original"
+    yield original_name
+    shutil.rmtree(cnst.directories['data'])
+
+
+def test_find_left_bound(get_image_greyscale):
+    assert 65 == iu.find_left_bound(get_image_greyscale)
+
+
+def test_find_bottom_bound(get_image_greyscale):
+    assert 1551 == iu.find_bottom_bound(get_image_greyscale)
+
+
+def test_find_right_bound(get_image_greyscale):
+    assert 688 == iu.find_right_bound(get_image_greyscale)
+
+
+def test_find_upper_bound(get_image_greyscale):
+    assert 9 == iu.find_upper_bound(get_image_greyscale)
+
+
+def test_crop_and_save_delete(get_image):
+    assert os.path.exists(get_image), "Verify the base image exists before function call"
+    _, name = iu.crop_and_save_psd(get_image)
+    assert not os.path.exists(get_image), "Verify the base image is removed after function call"
+    assert os.path.exists(name), "Verify the numpy image exists"
+
+
+def test_crop_and_save_resize(get_image):
+    default_size = (623, 1542)
+    size, _ = iu.crop_and_save_psd(get_image, delete_original=False)
+    assert size[0] == default_size[1], "Verify default resize"
+    assert size[1] == default_size[0], "Verify default resize"
+
+    resize_dimen = (5, 10)
+    size, _ = iu.crop_and_save_psd(get_image, delete_original=False, resize=True, resize_dimen=resize_dimen)
+    assert size[0] == resize_dimen[1], "Verify custom resize"
+    assert size[1] == resize_dimen[0], "Verify custom resize"
+
+    original_dimen = (623, 1542)
+    size, _ = iu.crop_and_save_psd(get_image, delete_original=False, resize=False)
+    assert size[0] == original_dimen[1], "Verify no resize"
+    assert size[1] == original_dimen[0], "Verify no resize"

satnogs_webscraper-1.4.4/tests/test_observation_dataset.py

@@ -0,0 +1,64 @@
+import os
+import shutil
+import pandas as pd
+import pytest
+import satnogs_webscraper.observation_dataset as od
+import satnogs_webscraper.constants as cnst
+from satnogs_webscraper.observation_scraper import ObservationScraper
+
+
+@pytest.fixture()
+def get_obs_ids():
+    if os.path.exists(cnst.directories['data']):
+        shutil.rmtree(cnst.directories['data'])
+    obs_list = ["5738648", "5740805", "5740806"]
+    cnst.verify_directories()
+    scraper = ObservationScraper(check_disk=False)
+    scraper.multiprocess_scrape_observations(obs_list)
+    yield obs_list
+
+    shutil.rmtree(cnst.directories['data'])
+
+
+def test_get_dataset(get_obs_ids):
+    df = od.get_dataset(get_obs_ids)
+    assert df['Observation_id'].iloc[0] == "5738648"
+    assert df['Observation_id'].iloc[1] == "5740805"
+    assert df['Observation_id'].iloc[2] == "5740806"
+
+
+def test_save_dataset(get_obs_ids):
+    save_name = "saved_observations.csv"
+
+    if os.path.exists(save_name):
+        os.remove(save_name)
+
+    od.save_dataset(get_obs_ids, save_name)
+
+    assert os.path.exists(save_name)
+
+    df = pd.read_csv(save_name)
+    assert df['Observation_id'].iloc[0] == 5738648
+    assert df['Observation_id'].iloc[1] == 5740805
+    assert df['Observation_id'].iloc[2] == 5740806
+
+
+def test_get_datasets(get_obs_ids):
+    meta_df, demod_df = od.get_datasets(get_obs_ids)
+
+    assert demod_df is None
+    assert meta_df.shape[0] == 3
+    assert meta_df.shape[1] == 13
+    assert '5740805' in meta_df.index
+    assert '5738648' in meta_df.index
+    assert '5740806' in meta_df.index
+
+    for idx in range(0,3):
+        assert meta_df.iloc[idx].meta_key == -1
+
+    for expected_column in ['Timeframe', 'Satellite', 'Station', 'Status', 'Status_Message',
+                            'Frequency', 'Mode', 'Metadata', 'Downloads', 'Waterfall_Status',
+                            'Polar_Plot', 'meta_key']:
+        assert expected_column in meta_df.columns
+
+

satnogs_webscraper-1.4.4/tests/test_observation_list_scraper.py

@@ -0,0 +1,122 @@
+import satnogs_webscraper.observation_list_scraper as ols
+import satnogs_webscraper.constants as cnst
+import pytest
+import os
+import shutil
+import re
+import json
+import requests
+
+
+@pytest.fixture()
+def prep_directories():
+    if os.path.exists(cnst.directories['data']):
+        shutil.rmtree(cnst.directories['data'])
+    cnst.verify_directories()
+    yield None
+    shutil.rmtree(cnst.directories['data'])
+
+
+def test_observation_list_scraper_init():
+    url = 'some_url'
+    save_name = "save_name"
+    save_dir = "some_dir"
+    page_limit = 2
+    resume = False
+    cpus = 2
+    fetcher1 = ols.ObservationListFetch(url, save_name, save_dir)
+    fetcher2 = ols.ObservationListFetch(url, save_name, save_dir, page_limit=page_limit, resume=resume, cpus=cpus)
+
+    assert url == fetcher1.url
+    assert save_name == fetcher1.save_name
+    assert save_dir == fetcher1.save_dir
+    assert fetcher1.page_limit is None
+    assert fetcher1.cpus is None
+    assert fetcher1.resume == True
+
+    assert url == fetcher2.url
+    assert save_name == fetcher2.save_name
+    assert save_dir == fetcher2.save_dir
+    assert page_limit == fetcher2.page_limit
+    assert resume == fetcher2.resume
+    assert cpus == fetcher2.cpus
+
+
+def test_get_pages(prep_directories):
+    test_url = "https://network.satnogs.org/observations/?future=0&bad=0&unknown=0&failed=0&norad=&observer=&station" \
+               "=&start=&end=&transmitter_mode="
+
+    fetcher1 = ols.ObservationListFetch(test_url, "", "", page_limit=2)
+
+    pages = fetcher1.get_pages(test_url)
+
+    assert len(pages) == 2
+
+    for page in pages:
+        assert 200 == requests.get(page).status_code
+
+
+def test_get_page_observation_ids(prep_directories):
+    test_url = "https://network.satnogs.org/observations/?future=0&bad=0&unknown=0&failed=0&norad=&observer=&station" \
+               "=&start=&end=&transmitter_mode="
+
+    fetcher1 = ols.ObservationListFetch(test_url, "", "", page_limit=2)
+    pages = fetcher1.get_pages(test_url)
+
+    ids_page0 = fetcher1.get_page_observation_ids(pages[0])
+    assert len(ids_page0) == 20
+    for id in ids_page0:
+        assert id == re.search(r"\d+", id)[0]
+
+    ids_page1 = fetcher1.get_page_observation_ids(pages[1])
+    assert len(ids_page1) == 20
+    for id in ids_page1:
+        assert id == re.search(r"\d+", id)[0]
+
+
+def test_multiprocess_id_fetch(prep_directories):
+    test_url = "https://network.satnogs.org/observations/?future=0&bad=0&unknown=0&failed=0&norad=&observer=&station" \
+               "=&start=&end=&transmitter_mode="
+    json1 = cnst.directories['observation_pages'] + "1.json"
+    json2 = cnst.directories['observation_pages'] + "2.json"
+
+    fetcher1 = ols.ObservationListFetch(test_url, "", page_limit=2)
+    fetcher1.multiprocess_id_fetch()
+
+    assert os.path.exists(json1)
+    assert os.path.exists(json2)
+
+    with open(json1) as fin:
+        json1_parsed = json.load(fin)
+
+    assert len(json1_parsed['IDs']) == 20
+    for id in json1_parsed['IDs']:
+        assert id == re.search(r"\d+", id)[0]
+
+    with open(json2) as fin:
+        json2_parsed = json.load(fin)
+
+    assert len(json2_parsed['IDs']) == 20
+    for id in json2_parsed['IDs']:
+        assert id == re.search(r"\d+", id)[0]
+
+
+def test_fetch_ids(prep_directories):
+    if os.path.exists("test.json"):
+        os.remove("test.json")
+
+    test_url = "https://network.satnogs.org/observations/?future=0&bad=0&unknown=0&failed=0&norad=&observer=&station" \
+               "=&start=&end=&transmitter_mode="
+
+    fetcher1 = ols.ObservationListFetch(test_url, "test.json", page_limit=2)
+
+    assert not os.path.exists("test.json")
+
+    ids = fetcher1.fetch_ids()
+
+    assert os.path.exists("test.json")
+
+    assert len(ids) == 40
+
+    for id in ids:
+        assert id == re.search(r"\d+", id)[0]

satnogs_webscraper-1.4.4/tests/test_observation_scraper.py

@@ -0,0 +1,197 @@
+import json
+import os.path
+import shutil
+import re
+import satnogs_webscraper.constants as cnst
+from satnogs_webscraper.observation_scraper import ObservationScraper
+import pytest
+
+record_7206380 = {
+    'Observation_id': '7206380',
+    'Timeframe': ['2023-02-25 21:08:54', '2023-02-25 21:13:09'],
+    'Satellite': '43880  - UWE-4',
+    'Station': '2802 - geoscan-2',
+    'Status': 'Good',
+    'Status_Message': '100',
+    'Frequency': '435,599,000Hz',
+    'Mode': 'FSK 9600',
+    'Metadata': {'radio': {'name': 'gr-satnogs',
+                           'version': 'v2.3-compat-xxx-v2.3.3.0',
+                           'parameters': {'soapy-rx-device': 'driver=airspy,biastee=false,bitpack=true',
+                                          'samp-rate-rx': '6e6',
+                                          'rx-freq': '435599000',
+                                          'file-path': '/tmp/.satnogs/data/receiving_satnogs_7206380_2023-02-25T21-08-54.out',
+                                          'waterfall-file-path': '/tmp/.satnogs/data/receiving_waterfall_7206380_2023-02-25T21-08-54.dat',
+                                          'decoded-data-file-path': '/tmp/.satnogs/data/data_7206380',
+                                          'doppler-correction-per-sec': None,
+                                          'lo-offset': None,
+                                          'ppm': None,
+                                          'rigctl-port': '4532',
+                                          'gain-mode': 'Settings Field',
+                                          'gain': None,
+                                          'antenna': 'RX',
+                                          'dev-args': None,
+                                          'stream-args': None,
+                                          'tune-args': None,
+                                          'other-settings': 'LNA=3,MIX=11,VGA=14',
+                                          'dc-removal': None,
+                                          'bb-freq': None,
+                                          'bw': None,
+                                          'enable-iq-dump': '1',
+                                          'iq-file-path': '/tmp/.satnogs/iq.bin',
+                                          'udp-dump-host': '127.0.0.1',
+                                          'udp-dump-port': 57356,
+                                          'wpm': None,
+                                          'baudrate': '9600',
+                                          'framing': 'ax25'}},
+                 'latitude': 60.002677,
+                 'longitude': 30.364098,
+                 'elevation': 50,
+                 'frequency': 435599000},
+    'Downloads': {
+        'audio': 'https://archive.org/download/satnogs-observations-007200001-007210000/satnogs-observations-007206001-007207000.zip/satnogs_7206380_2023-02-25T21-08-54.ogg',
+        'waterfall': 'https://s3.eu-central-1.wasabisys.com/satnogs-network/data_obs/2023/2/25/21/7206380/waterfall_7206380_2023-02-25T21-08-54.png',
+        'waterfall_hash_name': 'd4c1e87080299224b407920fc2ab56249b7961f28f4e57cfc565758f36250a6b',
+        'waterfall_shape': (1542, 623, 3)},
+    'Waterfall_Status': 'Unknown',
+    'Polar_Plot': {'tle1': '1 43880U 18111E   23056.19343271  .00005878  00000+0  47147-3 0  9994',
+                   'tle2': '2 43880  97.6498 323.1620 0014413 186.0212 174.0841 15.00395149227562',
+                   'timeframe-start': '2023-02-25T21:08:54+00:00',
+                   'timeframe-end': '2023-02-25T21:13:09+00:00',
+                   'groundstation-lat': '60.002677',
+                   'groundstation-lon': '30.364098',
+                   'groundstation-alt': '50'},
+    'demods': [{'original_name': 'data_obs/2023/2/25/21/7206380/data_7206380_2023-02-25T21-10-20_g0',
+                'location': '74851f09608f5c7f10a1d71460484c399b599a70352a58b1a990e3a7df3d13f0.bin'},
+               {'original_name': 'data_obs/2023/2/25/21/7206380/data_7206380_2023-02-25T21-10-28',
+                'location': 'c1126bc65bcfe09bbc97e574a7323c6b5f28db978e35b48c93a8b5f1e26505bd.bin'},
+               {'original_name': 'data_obs/2023/2/25/21/7206380/data_7206380_2023-02-25T21-10-47_g0',
+                'location': '0501c350339d0689505f8bcdf16236b2e1a7ecc6df90b706505c0f5f591d9773.bin'}]}
+
+
+@pytest.fixture()
+def prep_directories():
+    if os.path.exists(cnst.directories['data']):
+        shutil.rmtree(cnst.directories['data'])
+    cnst.verify_directories()
+    yield None
+    shutil.rmtree(cnst.directories['data'])
+
+
+def test_Observation_Scraper_init():
+    assert not os.path.exists(cnst.directories['data'])
+
+    fetch_waterfalls = True
+    fetch_logging = True
+    prints = True
+    check_disk = True
+    cpus = 1
+    grey_scale = True
+
+    obs_scraper = ObservationScraper()
+
+    assert os.path.exists(cnst.directories['data'])
+    assert fetch_waterfalls == obs_scraper.fetch_waterfalls
+    assert fetch_logging == obs_scraper.fetch_logging
+    assert prints == obs_scraper.prints
+    assert check_disk == obs_scraper.check_disk
+    assert cpus == obs_scraper.cpus
+    assert grey_scale == obs_scraper.grey_scale
+    assert isinstance(obs_scraper.observations_list, list)
+    assert cnst.files["observation_json"] == obs_scraper.json_file_loc
+    assert cnst.directories['observations'] == obs_scraper.observation_save_dir
+    assert cnst.files["log_file"] == obs_scraper.log_file_loc
+    assert cnst.directories['waterfalls'] == obs_scraper.waterfall_path
+    assert cnst.directories["demods"] == obs_scraper.demod_path
+
+    fetch_waterfalls = False
+    fetch_logging = False
+    prints = False
+    check_disk = False
+    cpus = 2
+    grey_scale = False
+
+    obs_scraper2 = ObservationScraper(fetch_waterfalls=fetch_waterfalls, fetch_logging=fetch_logging, prints=prints,
+                                      check_disk=check_disk, cpus=cpus, grey_scale=grey_scale)
+
+    assert fetch_waterfalls == obs_scraper2.fetch_waterfalls
+    assert fetch_logging == obs_scraper2.fetch_logging
+    assert prints == obs_scraper2.prints
+    assert check_disk == obs_scraper2.check_disk
+    assert cpus == obs_scraper2.cpus
+    assert grey_scale == obs_scraper2.grey_scale
+
+
+def test_observation_scrape(prep_directories):
+    observation_url = "https://network.satnogs.org/observations/7206380/"
+    obs_scraper = ObservationScraper()
+
+    scrape = obs_scraper.scrape_observation(url=observation_url)
+
+    keys_to_skip = ['demods', 'Metadata', 'Polar_Plot', 'Downloads', 'Timeframe']
+
+    # Test records that are in the table
+    for key in scrape.keys():
+        if key not in keys_to_skip:
+            assert scrape[key] == record_7206380[key]
+
+    # Test the timeframe record
+    assert 2 == len(scrape['Timeframe'])
+    assert scrape['Timeframe'][0] == record_7206380['Timeframe'][0]
+    assert scrape['Timeframe'][1] == record_7206380['Timeframe'][1]
+
+    # Test metadata
+    assert scrape['Metadata'] is not None
+    for key in record_7206380['Metadata'].keys():
+        assert record_7206380['Metadata'][key] == scrape['Metadata'][key]
+
+    # Test polar plot
+    assert scrape['Polar_Plot'] is not None
+    for key in record_7206380['Polar_Plot'].keys():
+        assert record_7206380['Polar_Plot'][key] == scrape['Polar_Plot'][key]
+
+    # Test the downloads
+    assert record_7206380['Downloads']['audio'] == scrape['Downloads']['audio']
+    assert record_7206380['Downloads']['waterfall'] == scrape['Downloads']['waterfall']
+    assert len(scrape['Downloads']['waterfall_shape']) == 2
+    assert scrape['Downloads']['waterfall_hash_name'].find(record_7206380['Downloads']['waterfall_hash_name']) != -1
+
+    # Test the demods
+    assert len(scrape['demods']) == len(record_7206380['demods'])
+    for s_demod, r_demod in zip(scrape['demods'], record_7206380['demods']):
+        assert s_demod['original_name'] == r_demod['original_name']
+        assert s_demod['location'].find(r_demod['location']) != -1
+
+
+def test_multi_process_observation_scrape(prep_directories):
+    obs_ids = [7206380, 7206656]
+    obs_names = [f'{obs_id}.json' for obs_id in obs_ids]
+    obs_scraper = ObservationScraper()
+    obs_scraper.multiprocess_scrape_observations(obs_ids)
+
+    scraped_files = os.listdir(cnst.directories['observations'])
+    for obs_name in obs_names:
+        assert obs_name in scraped_files
+
+    waterfall_files = [
+        "042cae91b4da57f450dda7a3504503c4d293ea9cbb268b087d51f484bae465e1.bin",
+        "d4c1e87080299224b407920fc2ab56249b7961f28f4e57cfc565758f36250a6b.bin"
+    ]
+
+    processed_waterfall_files = os.listdir(cnst.directories['waterfalls'])
+
+    for file in waterfall_files:
+        assert file in processed_waterfall_files
+
+    demod_files = [
+        "0501c350339d0689505f8bcdf16236b2e1a7ecc6df90b706505c0f5f591d9773.bin",
+        "74851f09608f5c7f10a1d71460484c399b599a70352a58b1a990e3a7df3d13f0.bin",
+        "830ecb5c37750234323882591ff8a0f3e5c3030a95a41e56c72c90eb64f5acec.bin",
+        "99f4e4cc9364e7f704219ed05289b273a2df8136438f78ebb7aaf6a4349dd109.bin",
+        "c1126bc65bcfe09bbc97e574a7323c6b5f28db978e35b48c93a8b5f1e26505bd.bin"
+    ]
+
+    processed_demod_files = os.listdir(cnst.directories['demods'])
+
+    for file in demod_files:
+        assert file in processed_demod_files

satnogs_webscraper-1.4.4/tests/test_request_utils.py

@@ -0,0 +1,73 @@
+import json
+import os.path
+import shutil
+import satnogs_webscraper.constants as cnst
+import satnogs_webscraper.request_utils as ru
+import pytest
+
+
+@pytest.fixture()
+def prep_directories():
+
+    if os.path.exists(cnst.directories['data']):
+        shutil.rmtree(cnst.directories['data'])
+    cnst.verify_directories()
+    yield None
+    shutil.rmtree(cnst.directories['data'])
+
+
+def test_write_log(prep_directories):
+    assert len(os.listdir(cnst.directories['logs'])) == 0
+
+    url = "test/url"
+    code = 123
+    comment = "This is a comment"
+    log_file = ru.write_log(url=url, code=code, comment=comment)
+
+    assert len(os.listdir(cnst.directories['logs'])) == 1
+
+    with open(log_file) as file_in:
+        parsed_logs = json.load(file_in)
+
+    assert parsed_logs['url'] == url
+    assert parsed_logs['status'] == code
+    assert parsed_logs['comment'] == comment
+
+
+def test_get_request_logging(prep_directories):
+    url = 'https://www.google.com/'
+
+    assert len(os.listdir(cnst.directories['logs'])) == 0
+
+    res = ru.get_request(url)
+
+    assert len(os.listdir(cnst.directories['logs'])) == 1
+
+    log_file = os.listdir(cnst.directories['logs'])[0]
+    log_file = os.path.join(cnst.directories['logs'], log_file)
+
+    with open(log_file) as file_in:
+        parsed_logs = json.load(file_in)
+
+    assert parsed_logs['url'] == url
+    assert parsed_logs['status'] == res.status_code
+
+
+def test_get_request_exception_handling(prep_directories):
+    url = 'https://127.0.0.1/'
+
+    assert len(os.listdir(cnst.directories['logs'])) == 0
+
+    res = ru.get_request(url, max_count=2)
+
+    assert len(os.listdir(cnst.directories['logs'])) == 1
+
+    log_file = os.listdir(cnst.directories['logs'])[0]
+    log_file = os.path.join(cnst.directories['logs'], log_file)
+
+    with open(log_file) as file_in:
+        parsed_logs = json.load(file_in)
+
+    assert parsed_logs['url'] == url
+    assert parsed_logs['status'] == -1
+    assert parsed_logs['comment'].find("Exception") != -1

satnogs_webscraper-1.4.4/tests/test_scraper.py

@@ -0,0 +1,136 @@
+import kaitaistruct
+import importlib
+import os.path
+import shutil
+import satnogs_webscraper.constants as cnst
+from satnogs_webscraper.scraper import Scraper
+import pandas as pd
+import pytest
+import subprocess
+
+@pytest.fixture()
+def prep_directories():
+    if os.path.exists(cnst.directories['data']):
+        shutil.rmtree(cnst.directories['data'])
+    cnst.verify_directories()
+    yield None
+    shutil.rmtree(cnst.directories['data'])
+
+@pytest.fixture()
+def get_kaitai_parser():
+
+    if os.path.exists(cnst.directories['data']):
+        shutil.rmtree(cnst.directories['data'])
+    cnst.verify_directories()
+
+    get_ksy = ["wget",
+               "https://gitlab.com/librespacefoundation/satnogs/satnogs-decoders/-/raw/master/ksy/armadillo.ksy"]
+    compile_ksy = ["kaitai-struct-compiler", "-t", "python", "armadillo.ksy"]
+
+    try:
+        result = subprocess.run(get_ksy, check=True, capture_output=True, text=True)
+        print("Wget output:")
+        print(result.stdout)
+    except subprocess.CalledProcessError as e:
+        print(f"Wget failed with error: {e.stderr}")
+
+    try:
+        result = subprocess.run(compile_ksy, check=True, capture_output=True, text=True)
+        print("Compile output:")
+        print(result.stdout)
+    except subprocess.CalledProcessError as e:
+        print(f"Command failed with error: {e.stderr}")
+
+    yield importlib.import_module('armadillo').Armadillo
+
+    shutil.rmtree(cnst.directories['data'])
+    os.remove("armadillo.py")
+    os.remove("armadillo.ksy")
+
+
+def test_scraper_init(prep_directories):
+    expected_string = 'https://network.satnogs.org/observations/?future=0&bad=0&unknown=0&failed=0&norad=&observer=&' \
+                      'station=&start=&end=&transmitter_mode='
+    scraper = Scraper()
+    assert scraper.generate_query_string() == expected_string
+
+    expected_string2 = 'https://network.satnogs.org/observations/?future=0&bad=0&unknown=0&failed=0&norad=44352&' \
+                       'observer=&station=&start=&end=&transmitter_mode='
+
+    scraper2 = Scraper(norad="44352")
+    assert scraper2.generate_query_string() == expected_string2
+    scraper3 = Scraper(norad="44352", kaitai_interface="Test")
+    assert scraper3.kaitai_interface == "Test"
+
+
+def test_scraper_scrape_contents(prep_directories):
+    page_limit = 1
+    scraper = Scraper(norad="44352", list_page_limit=page_limit)
+    meta_df, demod_df = scraper.scrape()
+
+    # Testing the demod df container
+    assert demod_df.shape[0] == 6
+    assert demod_df.shape[1] == 2
+    assert 'num_demods' in demod_df.columns
+    assert 'dataframe' in demod_df.columns
+    assert 272 in demod_df.index
+    # Testing the best demod df scraped
+    demod_272 = demod_df.loc[272].dataframe
+    assert demod_272.shape[0] == 20
+    assert demod_272.shape[1] == 273
+    assert 'meta_key' in demod_272.columns
+    for i in range(0, 272):
+        assert i in demod_272.columns
+
+    # Testing meta df
+    assert meta_df.shape[0] == 20
+    assert meta_df.shape[1] == 13
+    for expected_column in ['Timeframe', 'Satellite', 'Station', 'Status', 'Status_Message',
+                            'Frequency', 'Mode', 'Metadata', 'Downloads', 'Waterfall_Status',
+                            'Polar_Plot', 'meta_key']:
+        assert expected_column in meta_df.columns
+
+    # Testing right join
+
+    df_merged = pd.merge(meta_df, demod_272, on='meta_key', how='right')
+    assert df_merged.shape[0] == 20
+    assert df_merged.shape[1] == 285
+
+    for expected_column in ['Timeframe', 'Satellite', 'Station', 'Status', 'Status_Message',
+                            'Frequency', 'Mode', 'Metadata', 'Downloads', 'Waterfall_Status',
+                            'Polar_Plot', 'meta_key']:
+        assert expected_column in df_merged.columns
+
+    for i in range(0, 272):
+        assert i in df_merged.columns
+
+    for name in meta_df['Satellite']:
+        assert name.find("ARMADILLO") != -1
+        assert name.find("44352") != -1
+
+    assert len(meta_df['Satellite'].unique()) == 1
+    assert sum(meta_df['Status'] == 'Good') == page_limit * 20
+    assert sum(meta_df['Status'] == 'Bad') == 0
+
+    scraper2 = Scraper(good=False, bad=True, list_page_limit=2)
+    meta_df, demod_df = scraper2.scrape()
+    assert len(meta_df['Satellite'].unique()) != 1
+    assert sum(meta_df['Status'] == 'Bad') == 2 * 20
+    assert sum(meta_df['Status'] == 'Good') == 0
+
+
+def test_kaitai_struct_interface(get_kaitai_parser):
+    scraper = Scraper(norad="44352", data=1, save_name="Armadillo", list_page_limit=3,
+                         kaitai_interface=get_kaitai_parser)
+
+    meta_df, demod_df = scraper.scrape()
+
+    assert meta_df.shape[1] == 14
+    assert sum(meta_df['kaitai'].apply(len) > 0) > 10
+
+    kaitai_obj = meta_df[meta_df['kaitai'].apply(len) > 0].kaitai[0][0]
+
+    assert type(kaitai_obj._io) == kaitaistruct.KaitaiStream
+
+    assert kaitai_obj.ax25_frame is not None
+

satnogs_webscraper-1.4.2/satnogs_webscraper/observation_dataset.py

@@ -1,20 +0,0 @@
-import json
-import os
-import pandas as pd
-import satnogs_webscraper.constants as cnst
-
-
-def save_dataset(observation_list, save_name):
-    df = get_dataset(observation_list)
-    df.to_csv(save_name, index=False)
-
-
-def get_dataset(observation_list):
-    observations = []
-    for observation in observation_list:
-        file_name = os.path.join(cnst.directories['observations'], f'{observation}.json')
-        with open(file_name, "r") as file_in:
-            observations.append(json.load(file_in))
-
-    df = pd.DataFrame(observations)
-    return df

{satnogs_webscraper-1.4.2 → satnogs_webscraper-1.4.4}/satnogs_webscraper.egg-info/requires.txt

@@ -1,8 +1,8 @@
+pillow
+numpy
+requests
 bs4
-dask[complete]
 html5lib
-numpy
 pandas
-pillow
+dask[complete]
 pytest
-requests