sarpyx 0.1.5__py3-none-any.whl → 0.1.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. docs/examples/advanced/batch_processing.py +1 -1
  2. docs/examples/advanced/custom_processing_chains.py +1 -1
  3. docs/examples/advanced/performance_optimization.py +1 -1
  4. docs/examples/basic/snap_integration.py +1 -1
  5. docs/examples/intermediate/quality_assessment.py +1 -1
  6. outputs/baseline/20260205-234828/__init__.py +33 -0
  7. outputs/baseline/20260205-234828/main.py +493 -0
  8. outputs/final/20260205-234851/__init__.py +33 -0
  9. outputs/final/20260205-234851/main.py +493 -0
  10. sarpyx/__init__.py +2 -2
  11. sarpyx/algorithms/__init__.py +2 -2
  12. sarpyx/cli/__init__.py +1 -1
  13. sarpyx/cli/focus.py +3 -5
  14. sarpyx/cli/main.py +106 -7
  15. sarpyx/cli/shipdet.py +1 -1
  16. sarpyx/cli/worldsar.py +549 -0
  17. sarpyx/processor/__init__.py +1 -1
  18. sarpyx/processor/core/decode.py +43 -8
  19. sarpyx/processor/core/focus.py +104 -57
  20. sarpyx/science/__init__.py +1 -1
  21. sarpyx/sla/__init__.py +8 -0
  22. sarpyx/sla/metrics.py +101 -0
  23. sarpyx/{snap → snapflow}/__init__.py +1 -1
  24. sarpyx/snapflow/engine.py +6165 -0
  25. sarpyx/{snap → snapflow}/op.py +0 -1
  26. sarpyx/utils/__init__.py +1 -1
  27. sarpyx/utils/geos.py +652 -0
  28. sarpyx/utils/grid.py +285 -0
  29. sarpyx/utils/io.py +77 -9
  30. sarpyx/utils/meta.py +55 -0
  31. sarpyx/utils/nisar_utils.py +652 -0
  32. sarpyx/utils/rfigen.py +108 -0
  33. sarpyx/utils/wkt_utils.py +109 -0
  34. sarpyx/utils/zarr_utils.py +55 -37
  35. {sarpyx-0.1.5.dist-info → sarpyx-0.1.6.dist-info}/METADATA +9 -5
  36. {sarpyx-0.1.5.dist-info → sarpyx-0.1.6.dist-info}/RECORD +41 -32
  37. {sarpyx-0.1.5.dist-info → sarpyx-0.1.6.dist-info}/WHEEL +1 -1
  38. sarpyx-0.1.6.dist-info/licenses/LICENSE +201 -0
  39. sarpyx-0.1.6.dist-info/top_level.txt +4 -0
  40. tests/test_zarr_compat.py +35 -0
  41. sarpyx/processor/core/decode_v0.py +0 -0
  42. sarpyx/processor/core/decode_v1.py +0 -849
  43. sarpyx/processor/core/focus_old.py +0 -1550
  44. sarpyx/processor/core/focus_v1.py +0 -1566
  45. sarpyx/processor/core/focus_v2.py +0 -1625
  46. sarpyx/snap/engine.py +0 -633
  47. sarpyx-0.1.5.dist-info/top_level.txt +0 -2
  48. {sarpyx-0.1.5.dist-info → sarpyx-0.1.6.dist-info}/entry_points.txt +0 -0
@@ -1,849 +0,0 @@
1
- import argparse
2
- import logging
3
- import os
4
- import pickle
5
- from pathlib import Path
6
- from typing import Any, Dict, List, Tuple, Union, Optional
7
- import json
8
-
9
- import numpy as np
10
- import pandas as pd
11
- from s1isp.decoder import (
12
- EUdfDecodingMode,
13
- SubCommutatedDataDecoder,
14
- decode_stream,
15
- decoded_stream_to_dict,
16
- decoded_subcomm_to_dict
17
- )
18
- from . import code2physical as pt
19
- from ...utils import zarr_utils
20
-
21
- save_array_to_zarr = zarr_utils.save_array_to_zarr # Ensure zarr_utils is imported for saving arrays
22
-
23
- # Configure logging
24
- logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
25
- logger = logging.getLogger(__name__)
26
-
27
-
28
- def extract_echo_bursts(records: List[Any]) -> Tuple[List[List[Any]], List[int]]:
29
- """Extract echo bursts from radar records and return burst data with indexes.
30
-
31
- This function filters radar records to extract only echo signals, then groups
32
- them by number of quads to create separate bursts. It's specifically designed
33
- for stripmap mode which typically has two bursts.
34
-
35
- Args:
36
- records (List[Any]): List of records from ISP decoder containing radar data.
37
- Each record should have radar_configuration_support.ses.signal_type
38
- and radar_sample_count.number_of_quads attributes.
39
-
40
- Returns:
41
- Tuple[List[List[Any]], List[int]]: A tuple containing:
42
- - List of echo burst sublists grouped by number of quads
43
- - List of indexes indicating burst start/end positions in original records
44
-
45
- Raises:
46
- AssertionError: If records list is empty or no echo records are found.
47
- ValueError: If unable to extract proper burst structure.
48
- """
49
- assert records, 'Records list cannot be empty'
50
-
51
- signal_types = {'noise': 1, 'tx_cal': 8, 'echo': 0}
52
- echo_signal_type = signal_types['echo']
53
-
54
- # Filter echo records
55
- filtered_records = [
56
- record for record in records
57
- if record[1].radar_configuration_support.ses.signal_type == echo_signal_type
58
- ]
59
-
60
- assert filtered_records, 'No echo records found in the input data'
61
-
62
- # Find first echo record index in original records
63
- echo_start_idx = None
64
- for idx, record in enumerate(records):
65
- if record[1].radar_configuration_support.ses.signal_type == echo_signal_type:
66
- echo_start_idx = idx
67
- break
68
-
69
- assert echo_start_idx is not None, 'Echo start index not found'
70
-
71
- # Extract number of quads for burst grouping
72
- def get_number_of_quads(record: Any) -> int:
73
- """Extract number of quads from a record."""
74
- return record[1].radar_sample_count.number_of_quads
75
-
76
- # Get unique quad counts for burst separation
77
- first_nq = get_number_of_quads(filtered_records[0])
78
- last_nq = get_number_of_quads(filtered_records[-1])
79
-
80
- # Create unique list of quad counts
81
- unique_quad_counts = list(dict.fromkeys([first_nq, last_nq])) # Preserves order, removes duplicates
82
-
83
- # Group bursts by number of quads
84
- bursts = []
85
- for quad_count in unique_quad_counts:
86
- burst = [record for record in filtered_records if get_number_of_quads(record) == quad_count]
87
- if burst: # Only add non-empty bursts
88
- bursts.append(burst)
89
-
90
- assert bursts, 'No valid bursts found after filtering'
91
-
92
- # Calculate burst boundary indexes
93
- indexes = [echo_start_idx]
94
- current_idx = echo_start_idx
95
-
96
- for burst in bursts:
97
- current_idx += len(burst)
98
- indexes.append(current_idx)
99
-
100
- return bursts, indexes
101
-
102
-
103
- def save_pickle_file(file_path: Union[str, Path], data: Any) -> None:
104
- """Save data to a pickle file with robust error handling.
105
-
106
- Args:
107
- file_path (Union[str, Path]): Path where to save the pickle file.
108
- data (Any): Data object to be pickled and saved.
109
-
110
- Raises:
111
- OSError: If file cannot be written due to permissions or disk space.
112
- ValueError: If file_path is empty or invalid.
113
- """
114
- if not file_path:
115
- raise ValueError('File path cannot be empty')
116
-
117
- file_path = Path(file_path)
118
-
119
- # Ensure parent directory exists
120
- file_path.parent.mkdir(parents=True, exist_ok=True)
121
-
122
- try:
123
- with open(file_path, 'wb') as f:
124
- pickle.dump(data, f)
125
- logger.info(f'Successfully saved pickle file: {file_path}')
126
- except Exception as e:
127
- logger.error(f'Failed to save pickle file {file_path}: {e}')
128
- raise
129
-
130
-
131
- def extract_headers(file_path: Union[str, Path], mode: str = 's1isp', apply_transformations: bool = False) -> pd.DataFrame:
132
- """Extract metadata headers from radar file using specified decoding mode.
133
-
134
- Args:
135
- file_path (Union[str, Path]): Path to the radar data file.
136
- mode (str): Extraction mode. Currently only 's1isp' is fully supported.
137
- apply_transformations (bool): Whether to apply parameter transformations to convert raw values to physical units.
138
-
139
- Returns:
140
- pd.DataFrame: DataFrame containing the extracted metadata headers with optional transformations.
141
-
142
- Raises:
143
- ValueError: If mode is not supported or file_path is invalid.
144
- FileNotFoundError: If file_path does not exist.
145
- RuntimeError: If decoding fails or transformation fails.
146
- """
147
- supported_modes = ['richa', 's1isp']
148
- if mode not in supported_modes:
149
- raise ValueError(f"Mode must be one of {supported_modes}, got '{mode}'")
150
-
151
- file_path = Path(file_path)
152
- if not file_path.exists():
153
- raise FileNotFoundError(f'File not found: {file_path}')
154
-
155
- if mode == 'richa':
156
- raise NotImplementedError('Richa mode requires meta_extractor implementation')
157
-
158
- try:
159
- records, _, _ = decode_stream(
160
- str(file_path),
161
- udf_decoding_mode=EUdfDecodingMode.NONE,
162
- )
163
- headers_data = decoded_stream_to_dict(records, enum_value=True)
164
- metadata_df = pd.DataFrame(headers_data)
165
-
166
- # Apply parameter transformations if requested
167
- if apply_transformations:
168
- try:
169
- metadata_df = _apply_parameter_transformations(metadata_df)
170
- logger.info(f'Applied parameter transformations to {len(metadata_df)} records')
171
- except Exception as e:
172
- logger.warning(f'Failed to apply parameter transformations: {e}')
173
- # Continue with raw values if transformations fail
174
-
175
- logger.info(f'Successfully extracted {len(metadata_df)} header records')
176
- return metadata_df
177
-
178
- except Exception as e:
179
- logger.error(f'Failed to extract headers from {file_path}: {e}')
180
- raise RuntimeError(f'Header extraction failed: {e}') from e
181
-
182
-
183
- def decode_radar_file(input_file: Union[str, Path], apply_transformations: bool = False) -> Tuple[List[Dict[str, Any]], List[int]]:
184
- """Decode Sentinel-1 Level 0 radar file and extract burst data with ephemeris.
185
-
186
- This function performs complete decoding of a radar file, extracting both
187
- the radar echo data and associated metadata including ephemeris information.
188
-
189
- Args:
190
- input_file (Union[str, Path]): Path to the input Level 0 radar file.
191
- apply_transformations (bool): Whether to apply parameter transformations to convert raw values to physical units.
192
-
193
- Returns:
194
- Tuple[List[Dict[str, Any]], List[int]]: A tuple containing:
195
- - List of dictionaries with 'echo', 'metadata', and 'ephemeris' keys
196
- - List of indexes indicating burst positions in the original data
197
-
198
- Raises:
199
- FileNotFoundError: If input file does not exist.
200
- RuntimeError: If decoding process fails.
201
- """
202
- input_file = Path(input_file)
203
- if not input_file.exists():
204
- raise FileNotFoundError(f'Input file not found: {input_file}')
205
-
206
- logger.info(f'Starting decode process for: {input_file}')
207
-
208
- try:
209
- # Decode the stream with UDF data
210
- records, _, subcom_data_records = decode_stream(
211
- str(input_file),
212
- udf_decoding_mode=EUdfDecodingMode.DECODE,
213
- )
214
-
215
- logger.info(f'Decoded {len(records)} records from file')
216
-
217
- # Process subcommutated data for ephemeris
218
- if subcom_data_records:
219
- subcom_decoder = SubCommutatedDataDecoder()
220
- subcom_decoded = subcom_decoder.decode(subcom_data_records)
221
- subcom_dict = decoded_subcomm_to_dict(subcom_decoded=subcom_decoded)
222
- ephemeris_df = pd.DataFrame(subcom_dict)
223
- logger.info(f'Extracted ephemeris data with {len(ephemeris_df)} records')
224
- else:
225
- logger.warning('No subcommutated data found, creating empty ephemeris DataFrame')
226
- ephemeris_df = pd.DataFrame()
227
-
228
- # Extract echo bursts
229
- echo_bursts, burst_indexes = extract_echo_bursts(records)
230
- logger.info(f'Extracted {len(echo_bursts)} echo bursts')
231
-
232
- # Process each burst
233
- processed_bursts = []
234
- for i, burst in enumerate(echo_bursts):
235
- try:
236
- # Extract metadata for this burst
237
- headers_data = decoded_stream_to_dict(burst, enum_value=True)
238
- burst_metadata = pd.DataFrame(headers_data)
239
-
240
- # Apply transformations if requested
241
- if apply_transformations:
242
- burst_metadata = _apply_parameter_transformations(burst_metadata)
243
-
244
- # Extract radar data (UDF - User Data Field)
245
- radar_data = np.array([record.udf for record in burst])
246
-
247
- burst_dict = {
248
- 'echo': radar_data,
249
- 'metadata': burst_metadata,
250
- 'ephemeris': ephemeris_df
251
- }
252
- processed_bursts.append(burst_dict)
253
-
254
- logger.info(f'Processed burst {i}: {radar_data.shape} radar samples, '
255
- f'{len(burst_metadata)} metadata records')
256
-
257
- except Exception as e:
258
- logger.error(f'Failed to process burst {i}: {e}')
259
- raise RuntimeError(f'Burst processing failed for burst {i}') from e
260
-
261
- return processed_bursts, burst_indexes
262
-
263
- except Exception as e:
264
- logger.error(f'Decoding failed for {input_file}: {e}')
265
- raise RuntimeError(f'File decoding failed: {e}') from e
266
-
267
-
268
- class S1L0Decoder:
269
- """Minimal API for decoding Sentinel-1 Level 0 data files."""
270
-
271
- def __init__(self, log_level: int = logging.INFO):
272
- """Initialize the decoder with logging configuration.
273
-
274
- Args:
275
- log_level: Logging level (default: INFO)
276
- """
277
- logging.basicConfig(level=log_level)
278
- self.logger = logging.getLogger(__name__)
279
-
280
- def decode_file(
281
- self,
282
- input_file: Path | str,
283
- output_dir: Optional[Path | str] = None,
284
- headers_only: bool = False,
285
- save_to_zarr: bool = False,
286
- apply_transformations: bool = True,
287
- ) -> Dict[str, Any]:
288
- """Decode a Sentinel-1 Level 0 .dat file.
289
-
290
- Args:
291
- input_file: Path to the input .dat file
292
- output_dir: Directory to save processed data (optional)
293
- headers_only: If True, extract only headers for quick preview
294
- save_to_zarr: If True, save output in Zarr format with compression
295
- apply_transformations: If True, apply parameter transformations to convert raw values to physical units
296
-
297
- Returns:
298
- Dictionary containing decoded data with keys:
299
- - 'burst_data': List of burst dictionaries (if headers_only=False)
300
- - 'headers': DataFrame with header information (if headers_only=True)
301
- - 'file_info': Basic file information
302
-
303
- Raises:
304
- FileNotFoundError: If input file doesn't exist
305
- ValueError: If decoding fails
306
- """
307
- input_path = Path(input_file)
308
-
309
- if not input_path.exists():
310
- raise FileNotFoundError(f'Input file not found: {input_path}')
311
-
312
- self.logger.info(f'Processing file: {input_path}')
313
- self.logger.info(f'File size: {input_path.stat().st_size / (1024**2):.1f} MB')
314
-
315
- result = {
316
- 'file_info': {
317
- 'path': str(input_path),
318
- 'size_mb': input_path.stat().st_size / (1024**2),
319
- 'filename': input_path.name
320
- }
321
- }
322
-
323
- try:
324
- if headers_only:
325
- # Extract headers only for quick preview
326
- self.logger.info('Extracting headers only...')
327
- headers_df = extract_headers(input_path, mode='s1isp', apply_transformations=apply_transformations)
328
-
329
- result['headers'] = headers_df
330
- result['num_records'] = len(headers_df)
331
-
332
- self.logger.info(f'Extracted {len(headers_df)} header records')
333
-
334
- else:
335
- # Full decoding
336
- self.logger.info('Starting full decode process...')
337
- burst_data, burst_indexes = decode_radar_file(input_path, apply_transformations=apply_transformations)
338
-
339
- result['burst_data'] = burst_data
340
- result['burst_indexes'] = burst_indexes
341
- result['num_bursts'] = len(burst_data)
342
-
343
- # Add burst summaries
344
- burst_summaries = []
345
- for i, burst in enumerate(burst_data):
346
- summary = {
347
- 'burst_id': i,
348
- 'echo_shape': burst['echo'].shape,
349
- 'metadata_records': len(burst['metadata']),
350
- 'ephemeris_records': len(burst['ephemeris'])
351
- }
352
- burst_summaries.append(summary)
353
-
354
- result['burst_summaries'] = burst_summaries
355
-
356
- self.logger.info(f'Successfully decoded {len(burst_data)} bursts')
357
-
358
- # ------------ Save data ---------- if output directory is specified
359
- if output_dir is not None:
360
- if save_to_zarr:
361
- self.logger.info('Saving data in Zarr format...')
362
- save_path = self._save_data_zarr(result, input_path, Path(output_dir))
363
- else:
364
- self.logger.info('Saving data in pickle format...')
365
- save_path = self._save_data(result, input_path, Path(output_dir))
366
- result['saved_to'] = str(save_path)
367
- self.logger.info('\n' + '='*50 + ' ✅ Processed. ' + '='*50)
368
- return result
369
-
370
- except Exception as e:
371
- self.logger.error(f'Decoding failed: {e}')
372
- raise ValueError(f'Failed to decode file: {e}') from e
373
-
374
- def _save_data(
375
- self,
376
- decoded_data: Dict[str, Any],
377
- input_path: Path,
378
- output_dir: Path
379
- ) -> Path:
380
- """Save decoded data to pickle files.
381
-
382
- Args:
383
- decoded_data: Dictionary containing decoded data
384
- input_path: Original input file path
385
- output_dir: Directory to save files
386
-
387
- Returns:
388
- Path to the output directory
389
- """
390
- output_dir.mkdir(exist_ok=True, parents=True)
391
- file_stem = input_path.stem
392
-
393
- self.logger.info(f'Saving processed data to: {output_dir}')
394
-
395
- if 'headers' in decoded_data:
396
- # Save headers only
397
- headers_path = output_dir / f'{file_stem}_headers.pkl'
398
- decoded_data['headers'].to_pickle(headers_path)
399
- self.logger.info(f'Saved headers: {headers_path}')
400
-
401
- elif 'burst_data' in decoded_data:
402
- # Save full burst data
403
- burst_data = decoded_data['burst_data']
404
-
405
- # Save ephemeris (once for all bursts)
406
- if burst_data and not burst_data[0]['ephemeris'].empty:
407
- ephemeris_path = output_dir / f'{file_stem}_ephemeris.pkl'
408
- burst_data[0]['ephemeris'].to_pickle(ephemeris_path)
409
- self.logger.info(f'Saved ephemeris: {ephemeris_path}')
410
-
411
- # Save each burst
412
- for i, burst in enumerate(burst_data):
413
- # Save metadata
414
- metadata_path = output_dir / f'{file_stem}_burst_{i}_metadata.pkl'
415
- burst['metadata'].to_pickle(metadata_path)
416
-
417
- # Save radar echo data
418
- echo_path = output_dir / f'{file_stem}_burst_{i}_echo.pkl'
419
- save_pickle_file(echo_path, burst['echo'])
420
-
421
- self.logger.info(f'Saved burst {i}: metadata and echo data')
422
-
423
- # Save summary info
424
- info_path = output_dir / f'{file_stem}_info.pkl'
425
- summary_info = {k: v for k, v in decoded_data.items()
426
- if k not in ['burst_data', 'headers']}
427
- save_pickle_file(info_path, summary_info)
428
-
429
- total_files = len(list(output_dir.glob(f'{file_stem}*.pkl')))
430
- self.logger.info(f'Created {total_files} output files')
431
-
432
- return output_dir
433
-
434
- def _save_data_zarr(
435
- self,
436
- decoded_data: Dict[str, Any],
437
- input_path: Path,
438
- output_dir: Path,
439
- compressor_level: int = 9
440
- ) -> Path:
441
- """Save decoded data to Zarr files with compression.
442
-
443
- Args:
444
- decoded_data: Dictionary containing decoded data
445
- input_path: Original input file path
446
- output_dir: Directory to save files
447
- compressor_level: Compression level for Zarr (0-9, default: 9)
448
-
449
- Returns:
450
- Path to the output directory
451
- """
452
- output_dir.mkdir(exist_ok=True, parents=True)
453
- file_stem = input_path.stem
454
-
455
- self.logger.info(f'Start -> data to Zarr format: {output_dir}')
456
-
457
- if 'headers' in decoded_data:
458
- # Save headers as CSV since it's metadata only
459
- headers_path = output_dir / f'{file_stem}_headers.csv'
460
- decoded_data['headers'].to_csv(headers_path, index=False)
461
- self.logger.info(f'Saved headers as CSV: {headers_path}')
462
-
463
- elif 'burst_data' in decoded_data:
464
- # Save full burst data in Zarr format
465
- burst_data = decoded_data['burst_data']
466
-
467
- # Save each burst as separate Zarr array
468
- for i, burst in enumerate(burst_data):
469
- # Save radar echo data as Zarr with metadata and ephemeris
470
- echo_zarr_path = output_dir / f'{file_stem}_burst_{i}.zarr'
471
-
472
- try:
473
- save_array_to_zarr(
474
- array=burst['echo'],
475
- file_path=str(echo_zarr_path),
476
- compressor_level=compressor_level,
477
- parent_product=input_path.parent.name,
478
- metadata_df=burst['metadata'],
479
- ephemeris_df=burst['ephemeris'] if not burst['ephemeris'].empty else None
480
- )
481
-
482
- self.logger.info(f'Saved burst {i} echo data to Zarr: {echo_zarr_path}')
483
- self.logger.info(f' - Echo shape: {burst["echo"].shape}')
484
- self.logger.info(f' - Metadata records: {len(burst["metadata"])}')
485
- if not burst['ephemeris'].empty:
486
- self.logger.info(f' - Ephemeris records: {len(burst["ephemeris"])}')
487
-
488
- except Exception as e:
489
- self.logger.error(f'Failed to save burst {i} to Zarr: {e}')
490
- # Fallback to pickle for this burst
491
- echo_pkl_path = output_dir / f'{file_stem}_burst_{i}_echo.pkl'
492
- save_pickle_file(echo_pkl_path, burst['echo'])
493
-
494
- metadata_pkl_path = output_dir / f'{file_stem}_burst_{i}_metadata.pkl'
495
- burst['metadata'].to_pickle(metadata_pkl_path)
496
-
497
- self.logger.warning(f'Saved burst {i} as pickle files instead')
498
-
499
- # Save summary info as JSON for better readability
500
- info_path = output_dir / f'{file_stem}_info.json'
501
- summary_info = {k: v for k, v in decoded_data.items()
502
- if k not in ['burst_data', 'headers']}
503
-
504
- # Convert numpy types to native Python types for JSON serialization
505
- def convert_numpy_types(obj):
506
- if isinstance(obj, np.integer):
507
- return int(obj)
508
- elif isinstance(obj, np.floating):
509
- return float(obj)
510
- elif isinstance(obj, np.ndarray):
511
- return obj.tolist()
512
- elif isinstance(obj, dict):
513
- return {k: convert_numpy_types(v) for k, v in obj.items()}
514
- elif isinstance(obj, list):
515
- return [convert_numpy_types(item) for item in obj]
516
- return obj
517
-
518
- try:
519
- summary_info_converted = convert_numpy_types(summary_info)
520
- with open(info_path, 'w') as f:
521
- json.dump(summary_info_converted, f, indent=2)
522
- self.logger.info(f'Saved summary info as JSON: {info_path}')
523
- except Exception as e:
524
- # Fallback to pickle
525
- info_pkl_path = output_dir / f'{file_stem}_info.pkl'
526
- save_pickle_file(info_pkl_path, summary_info)
527
- self.logger.warning(f'Saved summary info as pickle: {info_pkl_path}')
528
-
529
- # Count total files created
530
- zarr_files = len(list(output_dir.glob('*.zarr')))
531
- other_files = len(list(output_dir.glob('*'))) - zarr_files
532
-
533
- self.logger.info(f'Created {zarr_files} Zarr arrays and {other_files} other files')
534
-
535
- return output_dir
536
-
537
- # Parameter Transformations Integration
538
- # =====================================
539
- # This module integrates parameter transformations from parameter_transformations.py
540
- # to convert raw bytecode values from Sentinel-1 data packets into meaningful physical
541
- # parameters. When apply_transformations=True is used:
542
- #
543
- # Key Physical Transformations Applied:
544
- # • fine_time: Raw 16-bit → Seconds using (raw + 0.5) * 2^-16
545
- # • rx_gain: Raw codes → dB using raw * -0.5
546
- # • pri: Raw counts → Seconds using raw / F_REF
547
- # • tx_pulse_length: Raw counts → Seconds using raw / F_REF
548
- # • tx_ramp_rate: Raw 16-bit → Hz/s using sign/magnitude extraction + F_REF² scaling
549
- # • tx_pulse_start_freq: Raw 16-bit → Hz using sign/magnitude + F_REF scaling
550
- # • range_decimation: Raw codes → Sample rate (Hz) using lookup table
551
- # • swst/swl: Raw counts → Seconds using raw / F_REF
552
- #
553
- # Additional Features:
554
- # • Validation columns (sync_marker_valid, baq_mode_valid, etc.)
555
- # • Descriptive columns (signal_type_name, polarization_name, etc.)
556
- # • Derived columns (samples_per_line, data_take_hex, etc.)
557
- # =====================================
558
-
559
-
560
- def _apply_parameter_transformations(metadata_df: pd.DataFrame) -> pd.DataFrame:
561
- """Apply parameter transformations using parameter_transformations.py API functions.
562
-
563
- This function integrates the parameter_transformations.py API directly to convert
564
- raw bytecode values from Sentinel-1 data packets into meaningful physical parameters
565
- using the exact mathematical transformations defined in the API.
566
-
567
- Args:
568
- metadata_df (pd.DataFrame): DataFrame containing raw metadata values from decoded packets.
569
-
570
- Returns:
571
- pd.DataFrame: DataFrame with transformed physical values and additional descriptive columns.
572
-
573
- Raises:
574
- Exception: If transformation of critical parameters fails.
575
- """
576
- transformed_df = metadata_df.copy()
577
-
578
- def safe_to_float(value: Any) -> Optional[float]:
579
- """Safely convert pandas scalar or other types to float, excluding complex numbers."""
580
- if value is None:
581
- return None
582
- if hasattr(value, '__array__') and pd.isna(value):
583
- return None
584
- if isinstance(value, complex):
585
- logger.warning(f'Cannot convert complex number to float: {value}')
586
- return None
587
-
588
- try:
589
- if isinstance(value, (np.number, np.ndarray)):
590
- return float(value.item())
591
- return float(value)
592
- except (ValueError, TypeError, OverflowError) as e:
593
- logger.warning(f'Failed to convert value to float: {value}, error: {e}')
594
- return None
595
-
596
- # Apply fine time transformation: (raw + 0.5) * 2^-16 → seconds
597
- try:
598
- if 'fine_time' in transformed_df.columns:
599
- transformed_df['fine_time'] = (transformed_df['fine_time'] + 0.5) * (2**-16)
600
- except Exception as e:
601
- logger.warning(f'Error applying fine time transformation: {e}')
602
-
603
- # Apply Rx gain transformation: raw * -0.5 → dB
604
- try:
605
- if 'rx_gain' in transformed_df.columns:
606
- transformed_df['rx_gain'] = transformed_df['rx_gain'] * -0.5
607
- except Exception as e:
608
- logger.warning(f'Error applying Rx gain transformation: {e}')
609
-
610
- # Apply Tx pulse ramp rate transformation: 16-bit sign/magnitude → Hz/s
611
- try:
612
- if 'tx_ramp_rate' in transformed_df.columns:
613
- def transform_txprr(raw_value: Any) -> Optional[float]:
614
- """Transform Tx pulse ramp rate using extract_tx_pulse_ramp_rate logic."""
615
- converted = safe_to_float(raw_value)
616
- if converted is None:
617
- return None
618
-
619
- tmp16 = int(converted)
620
- txprr_sign = (-1) ** (1 - (tmp16 >> 15))
621
- magnitude = tmp16 & 0x7FFF
622
- txprr = txprr_sign * magnitude * (pt.F_REF**2) / (2**21)
623
- return txprr
624
-
625
- transformed_df['tx_ramp_rate'] = transformed_df['tx_ramp_rate'].apply(transform_txprr)
626
- except Exception as e:
627
- logger.warning(f'Error applying Tx ramp rate transformation: {e}')
628
-
629
- # Apply Tx pulse start frequency transformation: 16-bit + TXPRR dependency → Hz
630
- try:
631
- if 'tx_pulse_start_freq' in transformed_df.columns and 'tx_ramp_rate' in metadata_df.columns:
632
- def transform_txpsf(row_idx: int) -> Optional[float]:
633
- """Transform Tx pulse start frequency using extract_tx_pulse_start_frequency logic."""
634
- raw_txpsf = transformed_df.loc[row_idx, 'tx_pulse_start_freq']
635
- raw_txprr = metadata_df.loc[row_idx, 'tx_ramp_rate']
636
-
637
- converted_txpsf = safe_to_float(raw_txpsf)
638
- converted_txprr = safe_to_float(raw_txprr)
639
-
640
- if converted_txpsf is None or converted_txprr is None:
641
- return None
642
-
643
- # Calculate TXPRR for additive component
644
- tmp16_txprr = int(converted_txprr)
645
- txprr_sign = (-1) ** (1 - (tmp16_txprr >> 15))
646
- txprr_magnitude = tmp16_txprr & 0x7FFF
647
- txprr = txprr_sign * txprr_magnitude * (pt.F_REF**2) / (2**21)
648
- txpsf_additive = txprr / (4 * pt.F_REF)
649
-
650
- # Extract TXPSF sign bit and magnitude
651
- tmp16_txpsf = int(converted_txpsf)
652
- txpsf_sign = (-1) ** (1 - (tmp16_txpsf >> 15))
653
- txpsf_magnitude = tmp16_txpsf & 0x7FFF
654
-
655
- # Apply scaling and combine components
656
- txpsf = txpsf_additive + txpsf_sign * txpsf_magnitude * pt.F_REF / (2**14)
657
- return txpsf
658
-
659
- transformed_df['tx_pulse_start_freq'] = [
660
- transform_txpsf(i) for i in transformed_df.index
661
- ]
662
- except Exception as e:
663
- logger.warning(f'Error applying Tx start frequency transformation: {e}')
664
-
665
- # Apply Tx pulse length transformation: raw / F_REF → seconds
666
- try:
667
- if 'tx_pulse_length' in transformed_df.columns:
668
- transformed_df['tx_pulse_length'] = transformed_df['tx_pulse_length'] / pt.F_REF
669
- except Exception as e:
670
- logger.warning(f'Error applying Tx pulse length transformation: {e}')
671
-
672
- # Apply PRI transformation: raw / F_REF → seconds
673
- try:
674
- if 'pri' in transformed_df.columns:
675
- transformed_df['pri'] = transformed_df['pri'] / pt.F_REF
676
- except Exception as e:
677
- logger.warning(f'Error applying PRI transformation: {e}')
678
-
679
- # Apply sampling window transformations: raw / F_REF → seconds
680
- try:
681
- if 'swst' in transformed_df.columns:
682
- transformed_df['swst'] = transformed_df['swst'] / pt.F_REF
683
-
684
- if 'swl' in transformed_df.columns:
685
- transformed_df['swl'] = transformed_df['swl'] / pt.F_REF
686
- except Exception as e:
687
- logger.warning(f'Error applying sampling window transformations: {e}')
688
-
689
- # -------- Removing Range Decimation Transformation --------
690
- # The range decimation transformation is commented out as it requires a specific API lookup table.
691
- # This is because the range decimation values are not directly convertible to sample rates
692
- # try:
693
- # if 'range_decimation' in transformed_df.columns:
694
- # transformed_df['range_decimation'] = transformed_df['range_decimation'].apply(
695
- # lambda x: pt.range_dec_to_sample_rate(int(x)) if pd.notna(x) else None
696
- # )
697
- # except Exception as e:
698
- # logger.warning(f'Error applying range decimation transformation: {e}')
699
- # -------- Removing Range Decimation Transformation --------
700
-
701
-
702
- # Apply additional descriptive transformations and validations from API
703
- try:
704
- # Signal type mapping
705
- if 'signal_type' in transformed_df.columns:
706
- signal_types = {0: 'echo', 1: 'noise', 8: 'tx_cal'}
707
- transformed_df['signal_type_name'] = transformed_df['signal_type'].map(signal_types)
708
-
709
- # Data take ID to hex representation
710
- if 'data_take_id' in transformed_df.columns:
711
- transformed_df['data_take_hex'] = transformed_df['data_take_id'].apply(
712
- lambda x: f'0x{int(x):08X}' if pd.notna(x) else None
713
- )
714
-
715
- # Number of quads to samples per line conversion
716
- if 'number_of_quads' in transformed_df.columns:
717
- transformed_df['samples_per_line'] = transformed_df['number_of_quads'] * 2
718
-
719
- # Polarization mapping
720
- if 'polarization' in transformed_df.columns:
721
- pol_mapping = {0: 'H', 1: 'V', 2: 'H+V', 3: 'H-V'}
722
- transformed_df['polarization_name'] = transformed_df['polarization'].map(pol_mapping)
723
-
724
- # Temperature compensation mapping
725
- if 'temperature_compensation' in transformed_df.columns:
726
- temp_comp_mapping = {0: 'disabled', 1: 'enabled', 2: 'reserved1', 3: 'reserved2'}
727
- transformed_df['temp_comp_name'] = transformed_df['temperature_compensation'].map(temp_comp_mapping)
728
-
729
- # Apply API validation functions
730
- if 'sync_marker' in transformed_df.columns:
731
- transformed_df['sync_marker_valid'] = transformed_df['sync_marker'].apply(
732
- lambda x: pt.validate_sync_marker(int(x)) if pd.notna(x) else False
733
- )
734
-
735
- if 'baq_mode' in transformed_df.columns:
736
- transformed_df['baq_mode_valid'] = transformed_df['baq_mode'].apply(
737
- lambda x: pt.validate_baq_mode(int(x)) if pd.notna(x) else False
738
- )
739
-
740
- if 'packet_version_number' in transformed_df.columns:
741
- transformed_df['packet_version_valid'] = transformed_df['packet_version_number'].apply(
742
- lambda x: pt.validate_packet_version(int(x)) if pd.notna(x) else False
743
- )
744
-
745
- except Exception as e:
746
- logger.warning(f'Error applying additional transformations: {e}')
747
-
748
- return transformed_df
749
-
750
-
751
-
752
-
753
- # ------------- Main Function -------------
754
- def main() -> int:
755
- """Main entry point for the radar file decoder.
756
-
757
- Handles command line arguments and orchestrates the decoding process.
758
-
759
- Returns:
760
- int: Exit code - 0 for success, 1 for failure.
761
- """
762
- parser = argparse.ArgumentParser(
763
- description='Decode Sentinel-1 Level 0 radar data files into processed bursts',
764
- formatter_class=argparse.RawDescriptionHelpFormatter,
765
- epilog="""
766
- Examples:
767
- python decode.py -i data.dat -o output_folder
768
- python decode.py --inputfile /path/to/radar.dat --output /path/to/output
769
- """,
770
- )
771
-
772
- parser.add_argument(
773
- '-i', '--inputfile',
774
- type=str,
775
- required=True,
776
- help='Path to input Level 0 radar data file (.dat)'
777
- )
778
- parser.add_argument(
779
- '-o', '--output',
780
- type=str,
781
- required=True,
782
- help='Path to output directory for processed files'
783
- )
784
- parser.add_argument(
785
- '-v', '--verbose',
786
- action='store_true',
787
- help='Enable verbose logging'
788
- )
789
-
790
- args = parser.parse_args()
791
-
792
- if args.verbose:
793
- logging.getLogger().setLevel(logging.DEBUG)
794
- logger.debug('Verbose logging enabled')
795
-
796
- # Validate inputs
797
- input_file = Path(args.inputfile)
798
- output_dir = Path(args.output)
799
-
800
- if not input_file.exists():
801
- logger.error(f'Input file does not exist: {input_file}')
802
- return 1
803
-
804
- # Create output directory
805
- try:
806
- output_dir.mkdir(parents=True, exist_ok=True)
807
- logger.info(f'Output directory ready: {output_dir}')
808
- except Exception as e:
809
- logger.error(f'Failed to create output directory {output_dir}: {e}')
810
- return 1
811
-
812
- # Process the file
813
- try:
814
- file_stem = input_file.stem
815
- logger.info(f'Processing file: {file_stem}')
816
-
817
- # Decode the radar file
818
- burst_data, _ = decode_radar_file(input_file, apply_transformations=False)
819
-
820
- # Save processed data
821
- ephemeris_saved = False
822
- for burst_idx, burst in enumerate(burst_data):
823
- # Save ephemeris once (it's the same for all bursts)
824
- if not ephemeris_saved and not burst['ephemeris'].empty:
825
- ephemeris_path = output_dir / f'{file_stem}_ephemeris.pkl'
826
- burst['ephemeris'].to_pickle(ephemeris_path)
827
- logger.info(f'Saved ephemeris data: {ephemeris_path}')
828
- ephemeris_saved = True
829
-
830
- # Save burst-specific metadata
831
- metadata_path = output_dir / f'{file_stem}_pkt_{burst_idx}_metadata.pkl'
832
- burst['metadata'].to_pickle(metadata_path)
833
-
834
- # Save radar data
835
- radar_data_path = output_dir / f'{file_stem}_pkt_{burst_idx}.pkl'
836
- save_pickle_file(radar_data_path, burst['echo'])
837
-
838
- logger.info(f'Saved burst {burst_idx} data: metadata and radar arrays')
839
-
840
- logger.info(f'Successfully processed {len(burst_data)} bursts from {input_file}')
841
- return 0
842
-
843
- except Exception as e:
844
- logger.error(f'Processing failed: {e}')
845
- return 1
846
-
847
-
848
- if __name__ == '__main__':
849
- exit(main())