undatum 1.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. undatum/__init__.py +8 -0
  2. undatum/__main__.py +24 -0
  3. undatum/ai/__init__.py +167 -0
  4. undatum/ai/base.py +104 -0
  5. undatum/ai/config.py +185 -0
  6. undatum/ai/perplexity.py +81 -0
  7. undatum/ai/providers.py +1327 -0
  8. undatum/ai/schemas.py +42 -0
  9. undatum/cmds/__init__.py +5 -0
  10. undatum/cmds/analyzer.py +574 -0
  11. undatum/cmds/cat.py +115 -0
  12. undatum/cmds/converter.py +668 -0
  13. undatum/cmds/counter.py +78 -0
  14. undatum/cmds/deduplicator.py +148 -0
  15. undatum/cmds/differ.py +148 -0
  16. undatum/cmds/doc.py +901 -0
  17. undatum/cmds/enumerator.py +84 -0
  18. undatum/cmds/excluder.py +120 -0
  19. undatum/cmds/exploder.py +92 -0
  20. undatum/cmds/filler.py +135 -0
  21. undatum/cmds/fixlengths.py +110 -0
  22. undatum/cmds/formatter.py +130 -0
  23. undatum/cmds/head.py +73 -0
  24. undatum/cmds/ingester.py +1857 -0
  25. undatum/cmds/joiner.py +210 -0
  26. undatum/cmds/query.py +73 -0
  27. undatum/cmds/renamer.py +105 -0
  28. undatum/cmds/replacer.py +110 -0
  29. undatum/cmds/reverser.py +103 -0
  30. undatum/cmds/sampler.py +128 -0
  31. undatum/cmds/schemer.py +739 -0
  32. undatum/cmds/searcher.py +110 -0
  33. undatum/cmds/selector.py +518 -0
  34. undatum/cmds/slicer.py +154 -0
  35. undatum/cmds/sniffer.py +150 -0
  36. undatum/cmds/sorter.py +186 -0
  37. undatum/cmds/statistics.py +871 -0
  38. undatum/cmds/table.py +96 -0
  39. undatum/cmds/tail.py +78 -0
  40. undatum/cmds/textproc.py +62 -0
  41. undatum/cmds/transformer.py +113 -0
  42. undatum/cmds/transposer.py +108 -0
  43. undatum/cmds/validator.py +147 -0
  44. undatum/common/__init__.py +8 -0
  45. undatum/common/filter.py +116 -0
  46. undatum/common/functions.py +80 -0
  47. undatum/common/iterable.py +291 -0
  48. undatum/common/schema_utils.py +325 -0
  49. undatum/common/scheme.py +262 -0
  50. undatum/constants.py +36 -0
  51. undatum/core.py +1527 -0
  52. undatum/formats/__init__.py +5 -0
  53. undatum/formats/docx.py +175 -0
  54. undatum/utils.py +325 -0
  55. undatum/validate/__init__.py +11 -0
  56. undatum/validate/commonrules.py +32 -0
  57. undatum/validate/ruscodes.py +201 -0
  58. undatum-1.1.1.dist-info/METADATA +1275 -0
  59. undatum-1.1.1.dist-info/RECORD +63 -0
  60. undatum-1.1.1.dist-info/WHEEL +5 -0
  61. undatum-1.1.1.dist-info/entry_points.txt +3 -0
  62. undatum-1.1.1.dist-info/licenses/LICENSE +21 -0
  63. undatum-1.1.1.dist-info/top_level.txt +1 -0
undatum/__init__.py ADDED
@@ -0,0 +1,8 @@
1
+ """
2
+ undatum: a command-line tool for data processing. Brings CSV simplicity to JSON lines and BSON
3
+
4
+ """
5
+
6
+ __version__ = "1.1.1"
7
+ __author__ = 'Ivan Begtin'
8
+ __licence__ = 'MIT'
undatum/__main__.py ADDED
@@ -0,0 +1,24 @@
1
+ #!/usr/bin/env python
2
+ """The main entry point. Invoke as `undatum' or `python -m undatum`.
3
+
4
+ This module provides the CLI entry point for the undatum package.
5
+ """
6
+ import sys
7
+
8
+ from .core import app
9
+
10
+
11
+ def main():
12
+ """Main entry point for the application.
13
+
14
+ Handles the CLI invocation and graceful shutdown on keyboard interrupt.
15
+ """
16
+ try:
17
+ app()
18
+ except KeyboardInterrupt:
19
+ print("Ctrl-C pressed. Aborting")
20
+ sys.exit(0)
21
+
22
+
23
+ if __name__ == '__main__':
24
+ main()
undatum/ai/__init__.py ADDED
@@ -0,0 +1,167 @@
1
+ """AI service module for dataset documentation."""
2
+ from typing import Any, Optional
3
+
4
+ from .base import AIAPIError, AIConfigurationError, AIService, AIServiceError
5
+ from .config import get_ai_config, get_provider_config
6
+ from .providers import (
7
+ LMStudioProvider,
8
+ OllamaProvider,
9
+ OpenAIProvider,
10
+ OpenRouterProvider,
11
+ PerplexityProvider,
12
+ )
13
+
14
+ # Provider registry
15
+ PROVIDERS = {
16
+ 'openai': OpenAIProvider,
17
+ 'openrouter': OpenRouterProvider,
18
+ 'ollama': OllamaProvider,
19
+ 'lmstudio': LMStudioProvider,
20
+ 'perplexity': PerplexityProvider,
21
+ }
22
+
23
+
24
+ def get_ai_service(provider: Optional[str] = None,
25
+ config: Optional[dict[str, Any]] = None) -> AIService:
26
+ """Get AI service instance based on configuration.
27
+
28
+ Args:
29
+ provider: Provider name (openai, openrouter, ollama, lmstudio, perplexity)
30
+ If None, will be auto-detected from config
31
+ config: Optional configuration dictionary. If None, will load from
32
+ environment variables and config files
33
+
34
+ Returns:
35
+ Configured AI service instance
36
+
37
+ Raises:
38
+ AIConfigurationError: If provider is not configured or invalid
39
+
40
+ Examples:
41
+ >>> # Auto-detect from environment
42
+ >>> service = get_ai_service()
43
+
44
+ >>> # Explicit provider
45
+ >>> service = get_ai_service('openai', {'api_key': '...', 'model': 'gpt-4'})
46
+
47
+ >>> # From config file
48
+ >>> service = get_ai_service('ollama')
49
+ """
50
+ # Load configuration
51
+ full_config = get_ai_config(config or {})
52
+
53
+ # Determine provider
54
+ if provider:
55
+ provider_name = provider.lower()
56
+ else:
57
+ provider_name = full_config.get('provider', '').lower()
58
+
59
+ # Backward compatibility: if PERPLEXITY_API_KEY is set and no provider specified
60
+ if not provider_name:
61
+ import os
62
+ if os.getenv('PERPLEXITY_API_KEY'):
63
+ provider_name = 'perplexity'
64
+ full_config['provider'] = 'perplexity'
65
+
66
+ if not provider_name:
67
+ raise AIConfigurationError(
68
+ "No AI provider specified. Set UNDATUM_AI_PROVIDER environment variable, "
69
+ "configure in undatum.yaml, or pass provider argument."
70
+ )
71
+
72
+ if provider_name not in PROVIDERS:
73
+ raise AIConfigurationError(
74
+ f"Unknown provider: {provider_name}. "
75
+ f"Available providers: {', '.join(PROVIDERS.keys())}"
76
+ )
77
+
78
+ # Get provider class
79
+ provider_class = PROVIDERS[provider_name]
80
+
81
+ # Get provider-specific configuration
82
+ provider_config = get_provider_config(full_config, provider_name)
83
+
84
+ # Instantiate provider
85
+ try:
86
+ return provider_class(**provider_config)
87
+ except AIConfigurationError as e:
88
+ raise AIConfigurationError(
89
+ f"Failed to configure {provider_name} provider: {str(e)}"
90
+ ) from e
91
+
92
+
93
+ # Backward compatibility: export old function signatures
94
+ def get_fields_info(fields, language='English', ai_service: Optional[AIService] = None):
95
+ """Get field descriptions (backward compatibility wrapper).
96
+
97
+ Args:
98
+ fields: List of field names or comma-separated string
99
+ language: Language for descriptions
100
+ ai_service: Optional AI service instance. If None, will auto-detect.
101
+
102
+ Returns:
103
+ Dictionary mapping field names to descriptions
104
+ """
105
+ if ai_service is None:
106
+ ai_service = get_ai_service()
107
+
108
+ # Handle both list and string input
109
+ if isinstance(fields, str):
110
+ fields = [f.strip() for f in fields.split(',')]
111
+
112
+ return ai_service.get_fields_info(fields, language)
113
+
114
+
115
+ def get_description(data, language='English', ai_service: Optional[AIService] = None):
116
+ """Get dataset description (backward compatibility wrapper).
117
+
118
+ Args:
119
+ data: Sample data as CSV string
120
+ language: Language for description
121
+ ai_service: Optional AI service instance. If None, will auto-detect.
122
+
123
+ Returns:
124
+ String description of the dataset
125
+ """
126
+ if ai_service is None:
127
+ ai_service = get_ai_service()
128
+
129
+ return ai_service.get_description(data, language)
130
+
131
+
132
+ def get_structured_metadata(data, fields, language='English', ai_service: Optional[AIService] = None):
133
+ """Get structured metadata (backward compatibility wrapper).
134
+
135
+ Args:
136
+ data: Sample data as CSV string
137
+ fields: List of field names
138
+ language: Language for descriptions
139
+ ai_service: Optional AI service instance. If None, will auto-detect.
140
+
141
+ Returns:
142
+ Dictionary with structured metadata fields
143
+ """
144
+ if ai_service is None:
145
+ ai_service = get_ai_service()
146
+
147
+ if isinstance(fields, str):
148
+ fields = [f.strip() for f in fields.split(',')]
149
+
150
+ return ai_service.get_structured_metadata(data, fields, language)
151
+
152
+
153
+ __all__ = [
154
+ 'AIService',
155
+ 'AIServiceError',
156
+ 'AIConfigurationError',
157
+ 'AIAPIError',
158
+ 'get_ai_service',
159
+ 'get_fields_info',
160
+ 'get_description',
161
+ 'get_structured_metadata',
162
+ 'OpenAIProvider',
163
+ 'OpenRouterProvider',
164
+ 'OllamaProvider',
165
+ 'LMStudioProvider',
166
+ 'PerplexityProvider',
167
+ ]
undatum/ai/base.py ADDED
@@ -0,0 +1,104 @@
1
+ """Base AI service interface for dataset documentation."""
2
+ from abc import ABC, abstractmethod
3
+ from typing import Optional
4
+
5
+
6
+ class AIServiceError(Exception):
7
+ """Base exception for AI service errors."""
8
+ pass
9
+
10
+
11
+ class AIConfigurationError(AIServiceError):
12
+ """Raised when AI service configuration is invalid."""
13
+ pass
14
+
15
+
16
+ class AIAPIError(AIServiceError):
17
+ """Raised when AI API call fails."""
18
+ def __init__(self, message: str, status_code: Optional[int] = None, response: Optional[str] = None):
19
+ super().__init__(message)
20
+ self.status_code = status_code
21
+ self.response = response
22
+
23
+
24
+ class AIService(ABC):
25
+ """Abstract base class for AI service providers."""
26
+
27
+ def __init__(self, api_key: Optional[str] = None, base_url: Optional[str] = None,
28
+ model: Optional[str] = None, timeout: int = 30):
29
+ """Initialize AI service.
30
+
31
+ Args:
32
+ api_key: API key for authentication
33
+ base_url: Base URL for API endpoint (optional, provider-specific defaults)
34
+ model: Model name to use
35
+ timeout: Request timeout in seconds
36
+ """
37
+ self.api_key = api_key
38
+ self.base_url = base_url
39
+ self.model = model
40
+ self.timeout = timeout
41
+
42
+ @abstractmethod
43
+ def get_fields_info(self, fields: list[str], language: str = 'English') -> dict[str, str]:
44
+ """Get descriptions for a list of field names.
45
+
46
+ Args:
47
+ fields: List of field names to describe
48
+ language: Language for descriptions (default: 'English')
49
+
50
+ Returns:
51
+ Dictionary mapping field names to their descriptions
52
+
53
+ Raises:
54
+ AIConfigurationError: If service is not properly configured
55
+ AIAPIError: If API call fails
56
+ """
57
+ pass
58
+
59
+ @abstractmethod
60
+ def get_description(self, data: str, language: str = 'English') -> str:
61
+ """Get a description of the dataset.
62
+
63
+ Args:
64
+ data: Sample data as CSV string
65
+ language: Language for description (default: 'English')
66
+
67
+ Returns:
68
+ String description of the dataset
69
+
70
+ Raises:
71
+ AIConfigurationError: If service is not properly configured
72
+ AIAPIError: If API call fails
73
+ """
74
+ pass
75
+
76
+ @abstractmethod
77
+ def get_structured_metadata(self, data: str, fields: list[str],
78
+ language: str = 'English') -> dict:
79
+ """Get structured dataset metadata from sample data.
80
+
81
+ Args:
82
+ data: Sample data as CSV string
83
+ fields: List of field names
84
+ language: Language for descriptions (default: 'English')
85
+
86
+ Returns:
87
+ Dictionary with structured metadata fields
88
+
89
+ Raises:
90
+ AIConfigurationError: If service is not properly configured
91
+ AIAPIError: If API call fails
92
+ """
93
+ pass
94
+
95
+ def _validate_config(self) -> None:
96
+ """Validate that required configuration is present.
97
+
98
+ Raises:
99
+ AIConfigurationError: If configuration is invalid
100
+ """
101
+ if not self.api_key:
102
+ raise AIConfigurationError(f"API key is required for {self.__class__.__name__}")
103
+ if not self.model:
104
+ raise AIConfigurationError(f"Model is required for {self.__class__.__name__}")
undatum/ai/config.py ADDED
@@ -0,0 +1,185 @@
1
+ """Configuration management for AI services."""
2
+ import os
3
+ from pathlib import Path
4
+ from typing import Any, Optional
5
+
6
+ import yaml
7
+
8
+
9
+ def find_config_file() -> Optional[Path]:
10
+ """Find configuration file in standard locations.
11
+
12
+ Checks:
13
+ 1. Current directory: undatum.yaml
14
+ 2. Home directory: ~/.undatum/config.yaml
15
+
16
+ Returns:
17
+ Path to config file if found, None otherwise
18
+ """
19
+ # Check current directory
20
+ current_dir_config = Path.cwd() / 'undatum.yaml'
21
+ if current_dir_config.exists():
22
+ return current_dir_config
23
+
24
+ # Check home directory
25
+ home_config = Path.home() / '.undatum' / 'config.yaml'
26
+ if home_config.exists():
27
+ return home_config
28
+
29
+ return None
30
+
31
+
32
+ def load_config_file() -> dict[str, Any]:
33
+ """Load configuration from YAML file.
34
+
35
+ Returns:
36
+ Dictionary with configuration, empty dict if no file found
37
+ """
38
+ config_file = find_config_file()
39
+ if not config_file:
40
+ return {}
41
+
42
+ try:
43
+ with open(config_file, encoding='utf-8') as f:
44
+ config = yaml.safe_load(f) or {}
45
+ return config.get('ai', {})
46
+ except (yaml.YAMLError, OSError, KeyError):
47
+ return {}
48
+
49
+
50
+ def get_env_config() -> dict[str, Any]:
51
+ """Load configuration from environment variables.
52
+
53
+ Environment variables:
54
+ - UNDATUM_AI_PROVIDER: Provider name (openai, openrouter, ollama, lmstudio, perplexity)
55
+ - {PROVIDER}_API_KEY: API key for the provider
56
+ - OLLAMA_BASE_URL: Base URL for Ollama (defaults to http://localhost:11434)
57
+ - LMSTUDIO_BASE_URL: Base URL for LM Studio (defaults to http://localhost:1234/v1)
58
+
59
+ Returns:
60
+ Dictionary with configuration from environment
61
+ """
62
+ config = {}
63
+
64
+ provider = os.getenv('UNDATUM_AI_PROVIDER')
65
+ if provider:
66
+ config['provider'] = provider
67
+
68
+ # Check for provider-specific API keys
69
+ api_keys = {
70
+ 'openai': os.getenv('OPENAI_API_KEY'),
71
+ 'openrouter': os.getenv('OPENROUTER_API_KEY'),
72
+ 'perplexity': os.getenv('PERPLEXITY_API_KEY'),
73
+ }
74
+
75
+ # Use the first available API key if provider not specified
76
+ if not provider:
77
+ for prov, key in api_keys.items():
78
+ if key:
79
+ config['provider'] = prov
80
+ config['api_key'] = key
81
+ break
82
+ else:
83
+ # Use provider-specific key
84
+ key = api_keys.get(provider.lower())
85
+ if key:
86
+ config['api_key'] = key
87
+
88
+ # Provider-specific base URLs
89
+ ollama_url = os.getenv('OLLAMA_BASE_URL')
90
+ if ollama_url:
91
+ config['ollama_base_url'] = ollama_url
92
+
93
+ lmstudio_url = os.getenv('LMSTUDIO_BASE_URL')
94
+ if lmstudio_url:
95
+ config['lmstudio_base_url'] = lmstudio_url
96
+
97
+ return config
98
+
99
+
100
+ def merge_config(cli_config: Optional[dict[str, Any]] = None,
101
+ file_config: Optional[dict[str, Any]] = None,
102
+ env_config: Optional[dict[str, Any]] = None) -> dict[str, Any]:
103
+ """Merge configurations with precedence: CLI > File > Environment.
104
+
105
+ Args:
106
+ cli_config: Configuration from CLI arguments
107
+ file_config: Configuration from config file
108
+ env_config: Configuration from environment variables
109
+
110
+ Returns:
111
+ Merged configuration dictionary
112
+ """
113
+ if env_config is None:
114
+ env_config = get_env_config()
115
+ if file_config is None:
116
+ file_config = load_config_file()
117
+ if cli_config is None:
118
+ cli_config = {}
119
+
120
+ # Start with environment config (lowest precedence)
121
+ merged = env_config.copy()
122
+
123
+ # Override with file config
124
+ merged.update(file_config)
125
+
126
+ # Override with CLI config (highest precedence)
127
+ merged.update(cli_config)
128
+
129
+ return merged
130
+
131
+
132
+ def get_ai_config(cli_config: Optional[dict[str, Any]] = None) -> dict[str, Any]:
133
+ """Get AI configuration with proper precedence.
134
+
135
+ Args:
136
+ cli_config: Optional CLI configuration to override defaults
137
+
138
+ Returns:
139
+ Complete AI configuration dictionary
140
+ """
141
+ return merge_config(cli_config=cli_config)
142
+
143
+
144
+ def get_provider_config(config: dict[str, Any], provider: str) -> dict[str, Any]:
145
+ """Extract provider-specific configuration.
146
+
147
+ Args:
148
+ config: Full configuration dictionary
149
+ provider: Provider name
150
+
151
+ Returns:
152
+ Provider-specific configuration
153
+ """
154
+ # Map providers to their environment variable API keys
155
+ provider_api_keys = {
156
+ 'openai': os.getenv('OPENAI_API_KEY'),
157
+ 'openrouter': os.getenv('OPENROUTER_API_KEY'),
158
+ 'perplexity': os.getenv('PERPLEXITY_API_KEY'),
159
+ }
160
+
161
+ # Determine API key: use provider-specific env var if available, otherwise use config
162
+ api_key = None
163
+ if provider.lower() in provider_api_keys:
164
+ # Always prefer provider-specific environment variable
165
+ api_key = provider_api_keys[provider.lower()]
166
+
167
+ # Fall back to config api_key if provider-specific env var not set
168
+ if not api_key:
169
+ api_key = config.get('api_key')
170
+
171
+ provider_config = {
172
+ 'api_key': api_key,
173
+ 'model': config.get('model'),
174
+ 'timeout': config.get('timeout', 30),
175
+ }
176
+
177
+ # Provider-specific base URLs
178
+ if provider == 'ollama':
179
+ provider_config['base_url'] = config.get('ollama_base_url') or config.get('base_url')
180
+ elif provider == 'lmstudio':
181
+ provider_config['base_url'] = config.get('lmstudio_base_url') or config.get('base_url')
182
+ else:
183
+ provider_config['base_url'] = config.get('base_url')
184
+
185
+ return provider_config
@@ -0,0 +1,81 @@
1
+ """AI-powered data analysis using Perplexity API."""
2
+ import csv
3
+ import os
4
+ import sys
5
+ from io import StringIO
6
+
7
+ import requests
8
+
9
+ PERPLEXITY_API_KEY = os.getenv('PERPLEXITY_API_KEY')
10
+
11
+
12
+ def find_between( s, first, last ):
13
+ try:
14
+ start = s.index( first ) + len( first )
15
+ end = s.index( last, start )
16
+ return s[start:end]
17
+ except ValueError:
18
+ return ""
19
+
20
+ def get_fields_info(fields, language='English'):
21
+ """Returns information about data fields"""
22
+ url = "https://api.perplexity.ai/chat/completions"
23
+ headers = {"Authorization": f"Bearer {PERPLEXITY_API_KEY}"}
24
+ payload = {
25
+ "model": "sonar",
26
+ "messages": [
27
+ {"role": "system", "content": "Be precise and concise, provide data output only CSV or JSON, accrording to request"},
28
+ {"role": "user", "content": (
29
+ f"Please describe in {language} these fields delimited by comma: {fields}"
30
+ "Please output as single csv table only with following fields: name and description"
31
+ )},
32
+ ],
33
+ "response_format": {
34
+ "type": "text",
35
+ },
36
+ }
37
+ response = requests.post(url, headers=headers, json=payload).json()
38
+ text = response["choices"][0]["message"]["content"]
39
+ a_text = find_between(text, "```csv", "```").strip()
40
+ if len(a_text) == 0:
41
+ a_text = find_between(text, "```", "```").strip()
42
+ f = StringIO()
43
+ f.write(a_text)
44
+ f.seek(0)
45
+ table = {}
46
+ dr = csv.reader(f, delimiter=',')
47
+ n = 0
48
+ for r in dr:
49
+ n += 1
50
+ if n == 1:
51
+ continue
52
+ table[r[0]] = r[1]
53
+ return table
54
+
55
+
56
+
57
+ def get_description(data, language='English'):
58
+ url = "https://api.perplexity.ai/chat/completions"
59
+ headers = {"Authorization": f"Bearer {PERPLEXITY_API_KEY}"}
60
+ payload = {
61
+ "model": "sonar",
62
+ "messages": [
63
+ {"role": "system", "content": "Be precise and concise, provide data output only CSV or JSON, accrording to request"},
64
+ {"role": "user", "content": (
65
+ f"""
66
+ I have the following CSV data:
67
+ {data}
68
+ Please provide short description in about this data in {language}. Consider this data as sample of the bigger dataset.Don't generate any code and data examples""")},
69
+ ],
70
+ "response_format": {
71
+ "type": "text",
72
+ },
73
+ }
74
+ response = requests.post(url, headers=headers, json=payload).json()
75
+ return response["choices"][0]["message"]["content"]
76
+
77
+
78
+
79
+
80
+ if __name__ == "__main__":
81
+ print(get_fields_info(sys.argv[1], sys.argv[2]))