catocli 2.1.4__py3-none-any.whl → 2.1.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of catocli might be problematic. Click here for more details.

@@ -1,7 +1,7 @@
1
1
  #!/usr/bin/env python3
2
2
  """
3
3
  Direct Terraform Import Script using Python
4
- Imports socket sites, WAN interfaces, and network ranges directly using subprocess calls to terraform import
4
+ Imports socket sites, WAN interfaces, LAN interfaces and network ranges directly using subprocess calls to terraform import
5
5
  Reads from JSON structure exported from Cato API
6
6
  Adapted from scripts/import_if_rules_to_tfstate.py for CLI usage
7
7
  """
@@ -12,10 +12,12 @@ import sys
12
12
  import re
13
13
  import time
14
14
  import glob
15
+ import csv
16
+ import os
17
+ import argparse
15
18
  from pathlib import Path
16
19
  from ..customLib import validate_terraform_environment
17
20
 
18
-
19
21
  def load_json_data(json_file):
20
22
  """Load socket sites data from JSON file"""
21
23
  try:
@@ -121,7 +123,10 @@ def extract_socket_sites_data(sites_data):
121
123
  'precedence': 'ACTIVE'
122
124
  })
123
125
 
124
- # Extract network ranges for this site (through LAN interfaces)
126
+ # Extract LAN interfaces and network ranges for this site
127
+ # Process LAN interfaces first to determine which ones will be created as separate resources
128
+ valid_lan_interfaces = []
129
+
125
130
  for lan_interface in site.get('lan_interfaces', []):
126
131
  interface_id = lan_interface.get('id', None)
127
132
  interface_name = lan_interface.get('name', None)
@@ -135,9 +140,20 @@ def extract_socket_sites_data(sites_data):
135
140
  interface_name = native_range.get('interface_name')
136
141
  interface_index = native_range.get('index')
137
142
 
138
- # print(f"Processing LAN interface: interface_name={interface_name}, interface_id={interface_id}, interface_index={interface_index}, default_lan={is_default_lan}")
139
- # Add LAN interfaces that have valid interface_id and interface_index (including default_lan interfaces)
140
- if interface_id!=None and interface_index!=None:
143
+ # Check if this interface matches the site's default native range (should be excluded)
144
+ native_range_data = site.get('native_range', {})
145
+ is_site_default_native = (
146
+ is_default_lan and
147
+ interface_index == native_range_data.get('index') and
148
+ interface_name == native_range_data.get('interface_name')
149
+ )
150
+
151
+ # Only include interfaces that:
152
+ # 1. Have both interface_index and interface_id
153
+ # 2. Are NOT the site's default native range interface
154
+ will_create_interface = (interface_index is not None and interface_id and not is_site_default_native)
155
+
156
+ if will_create_interface:
141
157
  # For default_lan interfaces, get additional info from the interface itself or native_range
142
158
  subnet = lan_interface.get('subnet', '')
143
159
  local_ip = lan_interface.get('local_ip', '')
@@ -160,37 +176,51 @@ def extract_socket_sites_data(sites_data):
160
176
  'local_ip': local_ip,
161
177
  'role': interface_index or interface_name,
162
178
  'site_name': site.get('name', ''),
179
+ 'is_default_lan': is_default_lan # Add this for debugging
163
180
  })
181
+
182
+ valid_lan_interfaces.append((interface_index, interface_name, interface_id, is_default_lan))
164
183
 
165
- for network_range in lan_interface.get('network_ranges', []):
166
- subnet = network_range.get('subnet')
167
- if network_range.get('id') and subnet and "native_range" not in network_range:
168
- # Use the same interface info logic for network ranges
169
- range_interface_id = interface_id
170
- range_interface_index = interface_index
171
- range_interface_name = interface_name
172
-
173
- # If this is a default_lan interface, use native_range info
174
- if is_default_lan:
175
- native_range = site.get('native_range', {})
176
- range_interface_id = native_range.get('interface_id')
177
- range_interface_name = native_range.get('interface_name')
178
- range_interface_index = native_range.get('index')
179
-
180
- # print(f"Processing Network Range subnet={subnet}, interface_id={range_interface_id}, network_range_id={network_range['id']}, default_lan={is_default_lan}")
181
- network_ranges.append({
182
- 'site_id': site['id'],
183
- 'site_name': site['name'],
184
- 'interface_id': range_interface_id, # Use actual interface ID, not index
185
- 'interface_index': range_interface_index, # Also pass interface index separately
186
- 'interface_name': range_interface_name,
187
- 'network_range_id': network_range['id'],
188
- 'name': network_range.get('rangeName', network_range.get('name', '')),
189
- 'subnet': subnet,
190
- 'vlan_tag': network_range.get('vlanTag', network_range.get('vlan', '')),
191
- 'range_type': 'VLAN' if (network_range.get('vlanTag') or network_range.get('vlan')) else 'Native',
192
- 'microsegmentation': network_range.get('microsegmentation', False)
193
- })
184
+ # Process network ranges for interfaces that will be created as separate resources
185
+ # OR for any interface that has network ranges (including virtual interfaces)
186
+ has_network_ranges = len(lan_interface.get('network_ranges', [])) > 0
187
+ should_process_ranges = will_create_interface or has_network_ranges
188
+
189
+ if should_process_ranges:
190
+ for network_range in lan_interface.get('network_ranges', []):
191
+ subnet = network_range.get('subnet')
192
+ if network_range.get('id') and subnet:
193
+ # Skip native ranges - these are managed at the site level, not as separate network range resources
194
+ is_native_range = network_range.get('native_range', False)
195
+ if is_native_range:
196
+ continue
197
+
198
+ # Use the same interface info logic for network ranges
199
+ range_interface_id = interface_id
200
+ range_interface_index = interface_index
201
+ range_interface_name = interface_name
202
+
203
+ # If this is a default_lan interface, use native_range info
204
+ if is_default_lan:
205
+ native_range = site.get('native_range', {})
206
+ range_interface_id = native_range.get('interface_id')
207
+ range_interface_name = native_range.get('interface_name')
208
+ range_interface_index = native_range.get('index')
209
+
210
+ # print(f"Processing Network Range subnet={subnet}, interface_index={range_interface_index}, network_range_id={network_range['id']}, will_create_interface={will_create_interface}")
211
+ network_ranges.append({
212
+ 'site_id': site['id'],
213
+ 'site_name': site['name'],
214
+ 'interface_id': range_interface_id, # Use actual interface ID, not index
215
+ 'interface_index': range_interface_index, # Also pass interface index separately
216
+ 'interface_name': range_interface_name,
217
+ 'network_range_id': network_range['id'],
218
+ 'name': network_range.get('rangeName', network_range.get('name', '')),
219
+ 'subnet': subnet,
220
+ 'vlan_tag': network_range.get('vlanTag', network_range.get('vlan', '')),
221
+ 'range_type': 'VLAN' if (network_range.get('vlanTag') or network_range.get('vlan')) else 'Native',
222
+ 'microsegmentation': network_range.get('microsegmentation', False)
223
+ })
194
224
 
195
225
  return sites, wan_interfaces, lan_interfaces, network_ranges
196
226
 
@@ -241,343 +271,6 @@ def run_terraform_import(resource_address, resource_id, timeout=60, verbose=Fals
241
271
  print(f"Unexpected error for {resource_address}: {e}")
242
272
  return False, "", str(e)
243
273
 
244
-
245
- # def find_rule_index(rules, rule_name):
246
- # """Find rule index by name."""
247
- # for index, rule in enumerate(rules):
248
- # if rule['name'] == rule_name:
249
- # return index
250
- # return None
251
-
252
-
253
- # def import_sections(sections, module_name, resource_type,
254
- # resource_name="sections", verbose=False):
255
- # """Import all sections"""
256
- # print("\nStarting section imports...")
257
- # total_sections = len(sections)
258
- # successful_imports = 0
259
- # failed_imports = 0
260
-
261
- # for i, section in enumerate(sections):
262
- # section_id = section['section_id']
263
- # section_name = section['section_name']
264
- # section_index = section['section_index']
265
- # resource_address = f'{module_name}.{resource_type}.{resource_name}["{section_name}"]'
266
- # print(f"\n[{i+1}/{total_sections}] Section: {section_name} (index: {section_index})")
267
-
268
- # # For sections, we use the section name as the ID since that's how Cato identifies them
269
- # success, stdout, stderr = run_terraform_import(resource_address, section_id, verbose=verbose)
270
-
271
- # if success:
272
- # successful_imports += 1
273
- # else:
274
- # failed_imports += 1
275
-
276
- # print(f"\nSection Import Summary: {successful_imports} successful, {failed_imports} failed")
277
- # return successful_imports, failed_imports
278
-
279
-
280
- # def import_rules(rules, module_name, verbose=False,
281
- # resource_type="cato_if_rule", resource_name="rules",
282
- # batch_size=10, delay_between_batches=2, auto_approve=False):
283
- # """Import all rules in batches"""
284
- # print("\nStarting rule imports...")
285
- # successful_imports = 0
286
- # failed_imports = 0
287
- # total_rules = len(rules)
288
-
289
- # for i, rule in enumerate(rules):
290
- # rule_id = rule['id']
291
- # rule_name = rule['name']
292
- # rule_index = find_rule_index(rules, rule_name)
293
- # terraform_key = sanitize_name_for_terraform(rule_name)
294
-
295
- # # Use array index syntax instead of rule ID
296
- # resource_address = f'{module_name}.{resource_type}.{resource_name}["{str(rule_name)}"]'
297
- # print(f"\n[{i+1}/{total_rules}] Rule: {rule_name} (index: {rule_index})")
298
-
299
- # success, stdout, stderr = run_terraform_import(resource_address, rule_id, verbose=verbose)
300
-
301
- # if success:
302
- # successful_imports += 1
303
- # else:
304
- # failed_imports += 1
305
-
306
- # # Ask user if they want to continue on failure (unless auto-approved)
307
- # if failed_imports <= 3 and not auto_approve: # Only prompt for first few failures
308
- # response = input(f"\nContinue with remaining imports? (y/n): ").lower()
309
- # if response == 'n':
310
- # print("Import process stopped by user.")
311
- # break
312
-
313
- # # Delay between batches
314
- # if (i + 1) % batch_size == 0 and i < total_rules - 1:
315
- # print(f"\n Batch complete. Waiting {delay_between_batches}s before next batch...")
316
- # time.sleep(delay_between_batches)
317
-
318
- # print(f"\n Rule Import Summary: {successful_imports} successful, {failed_imports} failed")
319
- # return successful_imports, failed_imports
320
-
321
-
322
- # def import_if_rules_to_tf(args, configuration):
323
- # """Main function to orchestrate the import process"""
324
- # try:
325
- # print(" Terraform Import Tool - Cato IFW Rules & Sections")
326
- # print("=" * 60)
327
-
328
- # # Load data
329
- # print(f" Loading data from {args.json_file}...")
330
- # policy_data = load_json_data(args.json_file)
331
-
332
- # # Extract rules and sections
333
- # rules, sections = extract_rules_and_sections(policy_data)
334
-
335
- # if hasattr(args, 'verbose') and args.verbose:
336
- # print(f"section_ids: {json.dumps(policy_data.get('section_ids', {}), indent=2)}")
337
-
338
- # print(f" Found {len(rules)} rules")
339
- # print(f" Found {len(sections)} sections")
340
-
341
- # if not rules and not sections:
342
- # print(" No rules or sections found. Exiting.")
343
- # return [{"success": False, "error": "No rules or sections found"}]
344
-
345
- # # Validate Terraform environment before proceeding
346
- # validate_terraform_environment(args.module_name, verbose=args.verbose)
347
-
348
- # # Ask for confirmation (unless auto-approved)
349
- # if not args.rules_only and not args.sections_only:
350
- # print(f"\n Ready to import {len(sections)} sections and {len(rules)} rules.")
351
- # elif args.rules_only:
352
- # print(f"\n Ready to import {len(rules)} rules only.")
353
- # elif args.sections_only:
354
- # print(f"\n Ready to import {len(sections)} sections only.")
355
-
356
- # if hasattr(args, 'auto_approve') and args.auto_approve:
357
- # print("\nAuto-approve enabled, proceeding with import...")
358
- # else:
359
- # confirm = input(f"\nProceed with import? (y/n): ").lower()
360
- # if confirm != 'y':
361
- # print("Import cancelled.")
362
- # return [{"success": False, "error": "Import cancelled by user"}]
363
-
364
- # total_successful = 0
365
- # total_failed = 0
366
-
367
- # # Import sections first (if not skipped)
368
- # if not args.rules_only and sections:
369
- # successful, failed = import_sections(sections, module_name=args.module_name, resource_type="cato_if_section", verbose=args.verbose)
370
- # total_successful += successful
371
- # total_failed += failed
372
-
373
- # # Import rules (if not skipped)
374
- # if not args.sections_only and rules:
375
- # successful, failed = import_rules(rules, module_name=args.module_name,
376
- # verbose=args.verbose, batch_size=args.batch_size,
377
- # delay_between_batches=args.delay,
378
- # auto_approve=getattr(args, 'auto_approve', False))
379
- # total_successful += successful
380
- # total_failed += failed
381
-
382
- # # Final summary
383
- # print("\n" + "=" * 60)
384
- # print(" FINAL IMPORT SUMMARY")
385
- # print("=" * 60)
386
- # print(f" Total successful imports: {total_successful}")
387
- # print(f" Total failed imports: {total_failed}")
388
- # print(f" Overall success rate: {(total_successful / (total_successful + total_failed) * 100):.1f}%" if (total_successful + total_failed) > 0 else "N/A")
389
- # print("\n Import process completed!")
390
-
391
- # return [{
392
- # "success": True,
393
- # "total_successful": total_successful,
394
- # "total_failed": total_failed,
395
- # "module_name": args.module_name
396
- # }]
397
-
398
- # except Exception as e:
399
- # print(f"ERROR: {str(e)}")
400
- # return [{"success": False, "error": str(e)}]
401
-
402
-
403
- # def load_wf_json_data(json_file):
404
- # """Load WAN Firewall data from JSON file"""
405
- # try:
406
- # with open(json_file, 'r') as f:
407
- # data = json.load(f)
408
- # return data['data']['policy']['wanFirewall']['policy']
409
- # except FileNotFoundError:
410
- # print(f"Error: JSON file '{json_file}' not found")
411
- # sys.exit(1)
412
- # except json.JSONDecodeError as e:
413
- # print(f"Error: Invalid JSON in '{json_file}': {e}")
414
- # sys.exit(1)
415
- # except KeyError as e:
416
- # print(f"Error: Expected JSON structure not found in '{json_file}': {e}")
417
- # sys.exit(1)
418
-
419
-
420
- # def import_wf_sections(sections, module_name, verbose=False,
421
- # resource_type="cato_wf_section", resource_name="sections"):
422
- # """Import all WAN Firewall sections"""
423
- # print("\nStarting WAN Firewall section imports...")
424
- # total_sections = len(sections)
425
- # successful_imports = 0
426
- # failed_imports = 0
427
-
428
- # for i, section in enumerate(sections):
429
- # section_id = section['section_id']
430
- # section_name = section['section_name']
431
- # section_index = section['section_index']
432
- # # Add module. prefix if not present
433
- # if not module_name.startswith('module.'):
434
- # module_name = f'module.{module_name}'
435
- # resource_address = f'{module_name}.{resource_type}.{resource_name}["{section_name}"]'
436
- # print(f"\n[{i+1}/{total_sections}] Section: {section_name} (index: {section_index})")
437
-
438
- # # For sections, we use the section name as the ID since that's how Cato identifies them
439
- # success, stdout, stderr = run_terraform_import(resource_address, section_id, verbose=verbose)
440
-
441
- # if success:
442
- # successful_imports += 1
443
- # else:
444
- # failed_imports += 1
445
-
446
- # print(f"\nWAN Firewall Section Import Summary: {successful_imports} successful, {failed_imports} failed")
447
- # return successful_imports, failed_imports
448
-
449
-
450
- # def import_wf_rules(rules, module_name, verbose=False,
451
- # resource_type="cato_wf_rule", resource_name="rules",
452
- # batch_size=10, delay_between_batches=2, auto_approve=False):
453
- # """Import all WAN Firewall rules in batches"""
454
- # print("\nStarting WAN Firewall rule imports...")
455
- # successful_imports = 0
456
- # failed_imports = 0
457
- # total_rules = len(rules)
458
-
459
- # for i, rule in enumerate(rules):
460
- # rule_id = rule['id']
461
- # rule_name = rule['name']
462
- # rule_index = find_rule_index(rules, rule_name)
463
- # terraform_key = sanitize_name_for_terraform(rule_name)
464
-
465
- # # Add module. prefix if not present
466
- # if not module_name.startswith('module.'):
467
- # module_name = f'module.{module_name}'
468
-
469
- # # Use array index syntax instead of rule ID
470
- # resource_address = f'{module_name}.{resource_type}.{resource_name}["{str(rule_name)}"]'
471
- # print(f"\n[{i+1}/{total_rules}] Rule: {rule_name} (index: {rule_index})")
472
-
473
- # success, stdout, stderr = run_terraform_import(resource_address, rule_id, verbose=verbose)
474
-
475
- # if success:
476
- # successful_imports += 1
477
- # else:
478
- # failed_imports += 1
479
-
480
- # # Ask user if they want to continue on failure (unless auto-approved)
481
- # if failed_imports <= 3 and not auto_approve: # Only prompt for first few failures
482
- # response = input(f"\nContinue with remaining imports? (y/n): ").lower()
483
- # if response == 'n':
484
- # print("Import process stopped by user.")
485
- # break
486
-
487
- # # Delay between batches
488
- # if (i + 1) % batch_size == 0 and i < total_rules - 1:
489
- # print(f"\n Batch complete. Waiting {delay_between_batches}s before next batch...")
490
- # time.sleep(delay_between_batches)
491
-
492
- # print(f"\nWAN Firewall Rule Import Summary: {successful_imports} successful, {failed_imports} failed")
493
- # return successful_imports, failed_imports
494
-
495
-
496
- # def import_wf_rules_to_tf(args, configuration):
497
- # """Main function to orchestrate the WAN Firewall import process"""
498
- # try:
499
- # print(" Terraform Import Tool - Cato WF Rules & Sections")
500
- # print("=" * 60)
501
-
502
- # # Load data
503
- # print(f" Loading data from {args.json_file}...")
504
- # policy_data = load_wf_json_data(args.json_file)
505
-
506
- # # Extract rules and sections
507
- # rules, sections = extract_rules_and_sections(policy_data)
508
-
509
- # if hasattr(args, 'verbose') and args.verbose:
510
- # print(f"section_ids: {json.dumps(policy_data.get('section_ids', {}), indent=2)}")
511
-
512
- # print(f" Found {len(rules)} rules")
513
- # print(f" Found {len(sections)} sections")
514
-
515
- # if not rules and not sections:
516
- # print(" No rules or sections found. Exiting.")
517
- # return [{"success": False, "error": "No rules or sections found"}]
518
-
519
- # # Add module. prefix if not present
520
- # module_name = args.module_name
521
- # if not module_name.startswith('module.'):
522
- # module_name = f'module.{module_name}'
523
- # # Validate Terraform environment before proceeding
524
- # validate_terraform_environment(module_name, verbose=args.verbose)
525
-
526
- # # Ask for confirmation (unless auto-approved)
527
- # if not args.rules_only and not args.sections_only:
528
- # print(f"\n Ready to import {len(sections)} sections and {len(rules)} rules.")
529
- # elif args.rules_only:
530
- # print(f"\n Ready to import {len(rules)} rules only.")
531
- # elif args.sections_only:
532
- # print(f"\n Ready to import {len(sections)} sections only.")
533
-
534
- # if hasattr(args, 'auto_approve') and args.auto_approve:
535
- # print("\nAuto-approve enabled, proceeding with import...")
536
- # else:
537
- # confirm = input(f"\nProceed with import? (y/n): ").lower()
538
- # if confirm != 'y':
539
- # print("Import cancelled.")
540
- # return [{"success": False, "error": "Import cancelled by user"}]
541
-
542
- # total_successful = 0
543
- # total_failed = 0
544
-
545
- # # Import sections first (if not skipped)
546
- # if not args.rules_only and sections:
547
- # successful, failed = import_wf_sections(sections, module_name=args.module_name, verbose=args.verbose)
548
- # total_successful += successful
549
- # total_failed += failed
550
-
551
- # # Import rules (if not skipped)
552
- # if not args.sections_only and rules:
553
- # successful, failed = import_wf_rules(rules, module_name=args.module_name,
554
- # verbose=args.verbose, batch_size=args.batch_size,
555
- # delay_between_batches=args.delay,
556
- # auto_approve=getattr(args, 'auto_approve', False))
557
- # total_successful += successful
558
- # total_failed += failed
559
-
560
- # # Final summary
561
- # print("\n" + "=" * 60)
562
- # print(" FINAL IMPORT SUMMARY")
563
- # print("=" * 60)
564
- # print(f" Total successful imports: {total_successful}")
565
- # print(f" Total failed imports: {total_failed}")
566
- # print(f" Overall success rate: {(total_successful / (total_successful + total_failed) * 100):.1f}%" if (total_successful + total_failed) > 0 else "N/A")
567
- # print("\n Import process completed!")
568
-
569
- # return [{
570
- # "success": True,
571
- # "total_successful": total_successful,
572
- # "total_failed": total_failed,
573
- # "module_name": args.module_name
574
- # }]
575
-
576
- # except Exception as e:
577
- # print(f"ERROR: {str(e)}")
578
- # return [{"success": False, "error": str(e)}]
579
-
580
-
581
274
  def import_socket_sites(sites, module_name, verbose=False,
582
275
  resource_type="cato_socket_site", resource_name="socket-site",
583
276
  batch_size=10, delay_between_batches=2, auto_approve=False):
@@ -685,7 +378,7 @@ def import_lan_interfaces(lan_interfaces, module_name, verbose=False,
685
378
 
686
379
  for i, interface in enumerate(lan_interfaces):
687
380
  site_id = interface['site_id']
688
- interface_id = interface['id']
381
+ interface_id = interface['id'] # Actual interface ID from CSV
689
382
  interface_index = interface['index']
690
383
  interface_name = interface['name']
691
384
  site_name = interface['site_name']
@@ -694,8 +387,8 @@ def import_lan_interfaces(lan_interfaces, module_name, verbose=False,
694
387
  if not module_name.startswith('module.'):
695
388
  module_name = f'module.{module_name}'
696
389
 
697
- # Updated addressing to use interface_index-based indexing:
698
- # module.sites.module.socket-site[site].module.lan_interfaces[interface_index].cato_lan_interface.interface[interface_id]
390
+ # Updated addressing to use interface_index-based indexing for resource addressing:
391
+ # module.sites.module.socket-site[site].module.lan_interfaces[interface_index].cato_lan_interface.interface[0]
699
392
  # Apply the same index formatting logic as the Terraform module
700
393
  try:
701
394
  # If index is a number, format as INT_X
@@ -705,10 +398,12 @@ def import_lan_interfaces(lan_interfaces, module_name, verbose=False,
705
398
  # If not a number or None, use as-is
706
399
  formatted_index = interface_index if interface_index else interface_id
707
400
 
401
+ # The resource address uses interface_index for both the module key and the for_each key
708
402
  resource_address = f'{module_name}.module.socket-site["{site_name}"].module.lan_interfaces["{formatted_index}"].cato_lan_interface.interface["{formatted_index}"]'
709
403
 
710
404
  print(f"\n[{i+1}/{total_interfaces}] LAN Interface: {interface_name} on {site_name} (Index: {interface_index}, ID: {interface_id})")
711
405
 
406
+ # Use the actual interface_id for importing, not the formatted index
712
407
  success, stdout, stderr = run_terraform_import(resource_address, interface_id, verbose=verbose)
713
408
 
714
409
  if success:
@@ -729,7 +424,7 @@ def import_lan_interfaces(lan_interfaces, module_name, verbose=False,
729
424
  print(f"\nLAN Interface Import Summary: {successful_imports} successful, {failed_imports} failed")
730
425
  return successful_imports, failed_imports
731
426
 
732
- def import_network_ranges(network_ranges, module_name, verbose=False,
427
+ def import_network_ranges(network_ranges, lan_interfaces, module_name, verbose=False,
733
428
  resource_type="cato_network_range", resource_name="network_range",
734
429
  batch_size=10, delay_between_batches=2, auto_approve=False):
735
430
  """Import all network ranges in batches"""
@@ -743,15 +438,12 @@ def import_network_ranges(network_ranges, module_name, verbose=False,
743
438
  range_name = network_range['name']
744
439
  site_name = network_range['site_name']
745
440
  subnet = network_range['subnet']
441
+ interface_index = network_range['interface_index']
746
442
 
747
443
  # Add module. prefix if not present
748
444
  if not module_name.startswith('module.'):
749
445
  module_name = f'module.{module_name}'
750
446
 
751
- # Use correct resource addressing for network ranges with interface_index-based addressing
752
- # module.sites.module.socket-site["site_name"].module.lan_interfaces["interface_index"].module.network_ranges.module.network_range["range_key"].cato_network_range.no_dhcp[0]
753
- interface_index = network_range['interface_index'] # Use interface index for addressing
754
-
755
447
  # Apply the same index formatting logic as the Terraform module
756
448
  try:
757
449
  # If index is a number, format as INT_X
@@ -761,14 +453,35 @@ def import_network_ranges(network_ranges, module_name, verbose=False,
761
453
  # If not a number or None, use as-is (fallback to interface_id if needed)
762
454
  formatted_index = interface_index if interface_index else network_range['interface_id']
763
455
 
764
- # Generate the same key format as the Terraform configuration:
765
- # "${network_range.interface_index}-${replace(network_range.name, " ", "_")}"
456
+ # Determine if this is a default interface range (connected to native/default interface)
457
+ # Check if this network range has a corresponding LAN interface resource that was extracted
458
+ # If no LAN interface was created for this interface_index, it's a default interface range
459
+ lan_interface_exists = any(
460
+ lan['index'] == interface_index for lan in lan_interfaces
461
+ if lan['site_name'] == site_name
462
+ )
463
+
464
+ is_default_interface = not lan_interface_exists
465
+
466
+ # Generate the correct key format based on whether this is a default interface range
766
467
  sanitized_range_name = range_name.replace(" ", "_")
767
- range_key = f"{formatted_index}-{sanitized_range_name}"
468
+ if is_default_interface:
469
+ # For default interface ranges, use "DEFAULT" as the prefix to match socket module logic
470
+ range_key = f"DEFAULT-{sanitized_range_name}"
471
+ else:
472
+ # For regular LAN interface ranges, use the formatted interface index
473
+ range_key = f"{formatted_index}-{sanitized_range_name}"
768
474
 
769
- resource_address = f'{module_name}.module.socket-site["{site_name}"].module.lan_interfaces["{formatted_index}"].module.network_ranges.module.network_range["{range_key}"].cato_network_range.no_dhcp[0]'
475
+ # Determine the correct resource addressing based on whether this is a default interface
476
+ if is_default_interface:
477
+ # Default interface network ranges are addressed directly under the socket-site module
478
+ resource_address = f'{module_name}.module.socket-site["{site_name}"].cato_network_range.default_interface_ranges["{range_key}"]'
479
+ else:
480
+ # Regular interface network ranges go through the lan_interfaces module
481
+ resource_address = f'{module_name}.module.socket-site["{site_name}"].module.lan_interfaces["{formatted_index}"].module.network_ranges.module.network_range["{range_key}"].cato_network_range.no_dhcp[0]'
770
482
 
771
- print(f"\n[{i+1}/{total_ranges}] Network Range: {range_name} - {subnet} ({network_range_id}) on {site_name} (ID: {network_range_id})")
483
+ print(f"\n[{i+1}/{total_ranges}] Network Range: {range_name} - {subnet} ({network_range_id}) on {site_name}")
484
+ print(f" Resource Address: {resource_address}")
772
485
 
773
486
  success, stdout, stderr = run_terraform_import(resource_address, network_range_id, verbose=verbose)
774
487
 
@@ -897,26 +610,81 @@ def import_socket_sites_to_tf(args, configuration):
897
610
  print(" Terraform Import Tool - Cato Socket Sites, WAN Interfaces & Network Ranges")
898
611
  print("=" * 80)
899
612
 
900
- # Load data
901
- print(f" Loading data from {args.json_file}...")
902
- sites_data = load_json_data(args.json_file)
613
+ # Determine data source and load data
614
+ data_type = getattr(args, 'data_type', None)
615
+ json_file = getattr(args, 'json_file', None) or getattr(args, 'json_file_legacy', None)
616
+ csv_file = getattr(args, 'csv_file', None)
617
+ csv_folder = getattr(args, 'csv_folder', None)
618
+
619
+ # Validate input arguments
620
+ if data_type:
621
+ # If data type is explicitly specified, validate corresponding file arguments
622
+ if data_type == 'json' and not json_file:
623
+ raise ValueError("--data-type json requires --json-file argument")
624
+ elif data_type == 'csv' and not csv_file:
625
+ raise ValueError("--data-type csv requires --csv-file argument")
626
+ elif data_type == 'json' and csv_file:
627
+ raise ValueError("Cannot specify both --data-type json and --csv-file")
628
+ elif data_type == 'csv' and json_file:
629
+ raise ValueError("Cannot specify both --data-type csv and --json-file")
630
+ else:
631
+ # Auto-detect data type if not specified
632
+ if json_file and csv_file:
633
+ raise ValueError("Cannot specify both JSON and CSV files. Use --data-type to specify which format to use.")
634
+ elif json_file and json_file.endswith('.json'):
635
+ data_type = 'json'
636
+ print(" Auto-detected JSON format from file extension")
637
+ elif csv_file and csv_file.endswith('.csv'):
638
+ data_type = 'csv'
639
+ print(" Auto-detected CSV format from file extension")
640
+ elif json_file:
641
+ data_type = 'json'
642
+ print(" Auto-detected JSON format")
643
+ elif csv_file:
644
+ data_type = 'csv'
645
+ print(" Auto-detected CSV format")
646
+ else:
647
+ print("\nERROR: No input file specified.\n")
648
+ print("Please provide either:")
649
+ print(" JSON: --json-file <file> or positional argument")
650
+ print(" CSV: --csv-file <file> [--csv-folder <folder>]\n")
651
+ print("Use 'catocli import socket_sites_to_tf -h' for detailed help and examples.")
652
+ raise ValueError("No input file provided")
653
+
654
+ # Validate inputs based on data type
655
+ if data_type == 'json':
656
+ if not json_file:
657
+ raise ValueError("JSON import requires --json-file or positional json_file argument")
658
+ print(f" Loading JSON data from {json_file}...")
659
+ sites_data = load_json_data(json_file)
660
+ elif data_type == 'csv':
661
+ if not csv_file:
662
+ raise ValueError("CSV import requires --csv-file argument")
663
+ print(f" Loading CSV data from {csv_file}...")
664
+ if csv_folder:
665
+ print(f" Loading network ranges from {csv_folder}...")
666
+ sites_data = load_csv_data(csv_file, csv_folder)
667
+ else:
668
+ raise ValueError(f"Unsupported data type: {data_type}")
903
669
 
904
670
  # Extract sites, WAN interfaces, LAN interfaces, and network ranges
905
671
  sites, wan_interfaces, lan_interfaces, network_ranges = extract_socket_sites_data(sites_data)
906
- # print("\n==================== DEBUG =====================\n")
907
- # print("sites",json.dumps( sites, indent=2))
908
- # print("wan_interfaces",json.dumps( wan_interfaces, indent=2))
909
- # print("lan_interfaces",json.dumps( lan_interfaces, indent=2))
910
- # print("network_ranges",json.dumps( network_ranges, indent=2))
911
- # print("\n==================== DEBUG =====================\n")
912
672
  if hasattr(args, 'verbose') and args.verbose:
673
+ print("\n==================== DEBUG =====================\n")
674
+ print("sites",json.dumps( sites, indent=2))
675
+ print("wan_interfaces",json.dumps( wan_interfaces, indent=2))
676
+ print("lan_interfaces",json.dumps( lan_interfaces, indent=2))
677
+ print("network_ranges",json.dumps( network_ranges, indent=2))
678
+ print("\n==================== DEBUG =====================\n")
913
679
  print(f"\nExtracted data summary:")
914
680
  print(f" Sites: {len(sites)}")
915
681
  print(f" WAN Interfaces: {len(wan_interfaces)}")
682
+ print(f" LAN Interfaces: {len(lan_interfaces)}")
916
683
  print(f" Network Ranges: {len(network_ranges)}")
917
684
 
918
685
  print(f" Found {len(sites)} sites")
919
686
  print(f" Found {len(wan_interfaces)} WAN interfaces")
687
+ print(f" Found {len(lan_interfaces)} LAN interfaces")
920
688
  print(f" Found {len(network_ranges)} network ranges")
921
689
 
922
690
  if not sites and not wan_interfaces and not network_ranges:
@@ -1012,7 +780,7 @@ def import_socket_sites_to_tf(args, configuration):
1012
780
 
1013
781
  # Import network ranges (if selected)
1014
782
  if (ranges_only or (not sites_only and not wan_only and not lan_only)) and network_ranges:
1015
- successful, failed = import_network_ranges(network_ranges, module_name=args.module_name,
783
+ successful, failed = import_network_ranges(network_ranges, lan_interfaces, module_name=args.module_name,
1016
784
  verbose=args.verbose, batch_size=args.batch_size,
1017
785
  delay_between_batches=args.delay,
1018
786
  auto_approve=getattr(args, 'auto_approve', False))
@@ -1042,3 +810,572 @@ def import_socket_sites_to_tf(args, configuration):
1042
810
  except Exception as e:
1043
811
  print(f"ERROR: {str(e)}")
1044
812
  return [{"success": False, "error": str(e)}]
813
+
814
+
815
+ def load_csv_data(csv_file, sites_config_dir=None):
816
+ """
817
+ Load socket sites data from CSV files
818
+
819
+ Args:
820
+ csv_file: Main sites CSV file
821
+ sites_config_dir: Directory containing network ranges CSV files
822
+
823
+ Returns:
824
+ List of sites in JSON format compatible with existing functions
825
+ """
826
+ try:
827
+ # Load main sites CSV and group by site
828
+ sites_dict = {}
829
+ with open(csv_file, 'r', newline='', encoding='utf-8') as f:
830
+ reader = csv.DictReader(f)
831
+ for row in reader:
832
+ if not row['site_name'].strip():
833
+ continue
834
+
835
+ site_name = row['site_name']
836
+ site_id = row['site_id'].strip()
837
+
838
+ # If this is the first row for this site (has site_id), create the site entry
839
+ if site_id and site_name not in sites_dict:
840
+ sites_dict[site_name] = {
841
+ 'id': site_id,
842
+ 'name': site_name,
843
+ 'description': row['site_description'],
844
+ 'type': row['site_type'],
845
+ 'connection_type': row['connection_type'],
846
+ 'site_location': {
847
+ 'countryCode': row['site_location_country_code'],
848
+ 'stateCode': row['site_location_state_code'],
849
+ 'city': row['site_location_city'],
850
+ 'address': row['site_location_address'],
851
+ 'timezone': row['site_location_timezone']
852
+ },
853
+ 'native_range': {
854
+ 'interface_id': row.get('native_range_interface_id', ''), # May not be in parent CSV
855
+ 'interface_name': row['native_range_interface_name'],
856
+ 'subnet': row['native_range_subnet'],
857
+ 'index': row['native_range_interface_index'],
858
+ 'range_name': row.get('native_range_name', ''),
859
+ 'range_id': row.get('native_range_id', ''),
860
+ 'vlan': row.get('native_range_vlan', None),
861
+ 'mdns_reflector': row['native_range_mdns_reflector'].upper() == 'TRUE' if row['native_range_mdns_reflector'] else False,
862
+ 'gateway': row['native_range_gateway'] or None,
863
+ 'range_type': row['native_range_type'],
864
+ 'translated_subnet': row['native_range_translated_subnet'] or None,
865
+ 'local_ip': row['native_range_local_ip'],
866
+ 'dhcp_settings': {
867
+ 'dhcp_type': row['native_range_dhcp_type'] or 'DHCP_DISABLED',
868
+ 'ip_range': row['native_range_dhcp_ip_range'] or None,
869
+ 'relay_group_id': row['native_range_dhcp_relay_group_id'] or None,
870
+ 'relay_group_name': row['native_range_dhcp_relay_group_name'] or None,
871
+ 'dhcp_microsegmentation': row['native_range_dhcp_microsegmentation'].upper() == 'TRUE' if row['native_range_dhcp_microsegmentation'] else False
872
+ }
873
+ },
874
+ 'wan_interfaces': [],
875
+ 'lan_interfaces': []
876
+ }
877
+
878
+ # Add default LAN interface from parent CSV native range data
879
+ # This ensures every site has its default LAN interface for import
880
+ # Note: interface_id may be provided later from site-specific CSV files
881
+ if row['native_range_interface_index']:
882
+ default_lan_interface = {
883
+ 'id': row.get('native_range_interface_id', ''), # May be empty, will be filled from site CSV
884
+ 'name': row['native_range_interface_name'],
885
+ 'index': row['native_range_interface_index'],
886
+ 'dest_type': 'LAN',
887
+ 'default_lan': True,
888
+ 'network_ranges': [] # Default interfaces typically don't have additional ranges
889
+ }
890
+ sites_dict[site_name]['lan_interfaces'].append(default_lan_interface)
891
+
892
+ # Add WAN interface from current row if WAN interface data exists
893
+ wan_id = row.get('wan_interface_id', '')
894
+ if wan_id.strip() and site_name in sites_dict:
895
+ wan_interface = {
896
+ 'id': wan_id,
897
+ 'index': row.get('wan_interface_index', ''),
898
+ 'name': row.get('wan_interface_name', ''),
899
+ 'upstream_bandwidth': int(row['wan_upstream_bw']) if row.get('wan_upstream_bw', '').strip() else 25,
900
+ 'downstream_bandwidth': int(row['wan_downstream_bw']) if row.get('wan_downstream_bw', '').strip() else 25,
901
+ 'dest_type': 'CATO', # Default, not available in current CSV format
902
+ 'role': row.get('wan_role', ''),
903
+ 'precedence': row.get('wan_precedence', 'ACTIVE')
904
+ }
905
+ sites_dict[site_name]['wan_interfaces'].append(wan_interface)
906
+
907
+ # Convert sites dictionary to list
908
+ sites = list(sites_dict.values())
909
+
910
+ # Load network ranges CSV files if sites_config_dir is provided
911
+ if sites_config_dir and os.path.exists(sites_config_dir):
912
+ # Get list of all CSV files in the directory
913
+ available_files = [f for f in os.listdir(sites_config_dir) if f.endswith('_network_ranges.csv')]
914
+
915
+ for site in sites:
916
+ site_name = site['name']
917
+ ranges_file_found = None
918
+
919
+ # Try different filename patterns to find the matching file
920
+ potential_names = [
921
+ site_name, # Exact name
922
+ site_name.replace(' ', '-'), # Spaces to dashes
923
+ site_name.replace(' ', '_'), # Spaces to underscores
924
+ site_name.replace('-', '_'), # Dashes to underscores
925
+ site_name.replace('/', '-'), # Slashes to dashes
926
+ site_name.replace('/', '_'), # Slashes to underscores
927
+ # Additional transformations for special cases
928
+ re.sub(r'[^a-zA-Z0-9_-]', '_', site_name), # Replace all special chars with underscores
929
+ re.sub(r'[^a-zA-Z0-9_-]', '-', site_name), # Replace all special chars with dashes
930
+ re.sub(r'[^a-zA-Z0-9]', '', site_name), # Remove all special chars
931
+ site_name.replace(' ', ''), # Remove all spaces
932
+ ]
933
+
934
+ # Look for matching file
935
+ for potential_name in potential_names:
936
+ expected_filename = f"{potential_name}_network_ranges.csv"
937
+ if expected_filename in available_files:
938
+ ranges_file_found = os.path.join(sites_config_dir, expected_filename)
939
+ break
940
+
941
+ if ranges_file_found:
942
+ load_site_network_ranges_csv(site, ranges_file_found)
943
+ else:
944
+ print(f"Warning: Network ranges file not found for site '{site_name}'. Tried: {[f'{name}_network_ranges.csv' for name in potential_names]}")
945
+ print(f" Available files: {available_files}")
946
+
947
+ return sites
948
+
949
+ except FileNotFoundError:
950
+ print(f"Error: CSV file '{csv_file}' not found")
951
+ sys.exit(1)
952
+ except Exception as e:
953
+ print(f"Error loading CSV data from '{csv_file}': {e}")
954
+ sys.exit(1)
955
+
956
+
957
+ def load_site_network_ranges_csv(site, ranges_csv_file):
958
+ """
959
+ Load network ranges for a site from CSV file and add to site data structure
960
+ New CSV structure:
961
+ - Rows with lan_interface_id create/define LAN interfaces
962
+ - Rows with network_range_id add network ranges to the current interface
963
+ - is_native_range indicates if a network range is native for that interface
964
+
965
+ Args:
966
+ site: Site dictionary to add ranges to
967
+ ranges_csv_file: Path to network ranges CSV file
968
+ """
969
+ try:
970
+ # Load CLI settings to get default interface mapping
971
+ from ....Utils.cliutils import load_cli_settings
972
+ settings = load_cli_settings()
973
+ # Note: load_cli_settings() now returns embedded defaults if file cannot be loaded
974
+ if not settings.get("default_socket_interface_map"):
975
+ print(f"Warning: No default socket interface mapping found for site {site['name']}")
976
+
977
+ with open(ranges_csv_file, 'r', newline='', encoding='utf-8') as f:
978
+ reader = csv.DictReader(f)
979
+
980
+ # Store interfaces by lan_interface_index for processing
981
+ interfaces = {}
982
+ current_interface_index = None
983
+ current_interface_data = None
984
+
985
+ for row in reader:
986
+ # Clean up row data (remove carriage returns)
987
+ cleaned_row = {k: v.strip() if isinstance(v, str) else v for k, v in row.items()}
988
+ row = cleaned_row
989
+
990
+ # Check if this row defines a LAN interface (has lan_interface_id)
991
+ has_lan_interface_id = bool(row.get('lan_interface_id', '').strip())
992
+ lan_interface_index = row.get('lan_interface_index', '').strip()
993
+
994
+ # Check if this is a default LAN interface (no interface ID but has index matching default)
995
+ is_default_interface = False
996
+ if not has_lan_interface_id and lan_interface_index:
997
+ connection_type = site.get('connection_type', '')
998
+ default_interface_index = settings.get("default_socket_interface_map", {}).get(connection_type)
999
+ if default_interface_index and lan_interface_index == default_interface_index:
1000
+ is_default_interface = True
1001
+
1002
+ # If this row has a LAN interface ID, create/update the interface
1003
+ # OR if this is a default interface, get details from parent CSV
1004
+ if (has_lan_interface_id or is_default_interface) and lan_interface_index:
1005
+
1006
+ # Create or get the interface data
1007
+ if lan_interface_index not in interfaces:
1008
+ if is_default_interface:
1009
+ # For default interfaces, get details from parent CSV native_range
1010
+ native_range = site.get('native_range', {})
1011
+ interfaces[lan_interface_index] = {
1012
+ 'id': native_range.get('interface_id', ''),
1013
+ 'name': native_range.get('interface_name', ''),
1014
+ 'index': lan_interface_index,
1015
+ 'dest_type': 'LAN', # Default for default interfaces
1016
+ 'default_lan': True, # Mark as default interface
1017
+ 'network_ranges': []
1018
+ }
1019
+ else:
1020
+ # For regular interfaces, get details from CSV row
1021
+ interfaces[lan_interface_index] = {
1022
+ 'id': row['lan_interface_id'],
1023
+ 'name': row.get('lan_interface_name', ''),
1024
+ 'index': lan_interface_index,
1025
+ 'dest_type': row.get('lan_interface_dest_type', 'LAN'),
1026
+ 'default_lan': False, # Will be determined by presence in native_range
1027
+ 'network_ranges': []
1028
+ }
1029
+
1030
+ current_interface_index = lan_interface_index
1031
+ current_interface_data = interfaces[lan_interface_index]
1032
+
1033
+ # If no new interface but we have a lan_interface_index, use existing interface
1034
+ elif lan_interface_index and lan_interface_index in interfaces:
1035
+ # This row continues with the same interface (no new lan_interface_id but same index)
1036
+ current_interface_index = lan_interface_index
1037
+ current_interface_data = interfaces[lan_interface_index]
1038
+
1039
+ # If this CSV row doesn't have a lan_interface_id but has network ranges,
1040
+ # mark the interface as virtual so network ranges get processed
1041
+ if not has_lan_interface_id and row.get('network_range_id', '').strip():
1042
+ current_interface_data['virtual_interface'] = True
1043
+ # If we have a lan_interface_index but no interface entry, create a virtual interface for processing network ranges
1044
+ elif lan_interface_index and row.get('network_range_id', '').strip():
1045
+ # Create a virtual interface entry for network range processing
1046
+ # This interface won't create a LAN interface resource, but allows network ranges to be processed
1047
+ if lan_interface_index not in interfaces:
1048
+ interfaces[lan_interface_index] = {
1049
+ 'id': None, # No interface resource will be created
1050
+ 'name': f"Virtual-{lan_interface_index}",
1051
+ 'index': lan_interface_index,
1052
+ 'dest_type': 'LAN',
1053
+ 'default_lan': False,
1054
+ 'network_ranges': [],
1055
+ 'virtual_interface': True # Mark as virtual
1056
+ }
1057
+ current_interface_index = lan_interface_index
1058
+ current_interface_data = interfaces[lan_interface_index]
1059
+
1060
+ # Also mark this interface as virtual if it wasn't created with an interface ID
1061
+ if not current_interface_data.get('id'):
1062
+ current_interface_data['virtual_interface'] = True
1063
+
1064
+ # Process network range data if present and we have a current interface
1065
+ if current_interface_data and row.get('network_range_id', '').strip():
1066
+
1067
+ network_range = {
1068
+ 'id': row['network_range_id'],
1069
+ 'name': row.get('network_range_name', ''),
1070
+ 'subnet': row.get('subnet', ''),
1071
+ 'vlan': int(row['vlan']) if row.get('vlan', '').strip() else None,
1072
+ 'mdns_reflector': row.get('mdns_reflector', '').upper() == 'TRUE',
1073
+ 'gateway': row.get('gateway') or None,
1074
+ 'range_type': row.get('range_type', ''),
1075
+ 'translated_subnet': row.get('translated_subnet') or None,
1076
+ 'local_ip': row.get('local_ip', ''),
1077
+ 'native_range': row.get('is_native_range', '').upper() == 'TRUE', # update this to support json native_range=true
1078
+ 'dhcp_settings': {
1079
+ 'dhcp_type': row.get('dhcp_type', '') or 'DHCP_DISABLED',
1080
+ 'ip_range': row.get('dhcp_ip_range') or None,
1081
+ 'relay_group_id': row.get('dhcp_relay_group_id') or None,
1082
+ 'relay_group_name': row.get('dhcp_relay_group_name') or None,
1083
+ 'dhcp_microsegmentation': row.get('dhcp_microsegmentation', '').upper() == 'TRUE'
1084
+ }
1085
+ }
1086
+
1087
+ # Add network range to current interface
1088
+ current_interface_data['network_ranges'].append(network_range)
1089
+
1090
+ # Check if this interface should be marked as default_lan
1091
+ # by checking if this is marked as a native range
1092
+ is_native_range = row.get('is_native_range', '').upper() == 'TRUE'
1093
+ if is_native_range:
1094
+ native_range = site.get('native_range', {})
1095
+ interface_matches_native = (
1096
+ current_interface_data['index'] == native_range.get('index') or
1097
+ current_interface_data['name'] == native_range.get('interface_name')
1098
+ )
1099
+ if interface_matches_native:
1100
+ current_interface_data['default_lan'] = True
1101
+ # IMPORTANT: Do not add this network range to the interface's network_ranges
1102
+ # because it's the site's native range and will be handled separately
1103
+ # Remove it from the network_ranges list - it was just added above
1104
+ current_interface_data['network_ranges'].pop()
1105
+ # Skip processing this network range further
1106
+ continue
1107
+
1108
+ # Add interfaces to site, but first merge with any existing default interface
1109
+ existing_interfaces = site.get('lan_interfaces', [])
1110
+ new_interfaces = list(interfaces.values())
1111
+
1112
+ # Check for default LAN interface conflicts and merge
1113
+ final_interfaces = []
1114
+ default_interface_found = False
1115
+
1116
+ # First add existing interfaces, updating any that match new interfaces
1117
+ for existing_interface in existing_interfaces:
1118
+ if existing_interface.get('default_lan', False):
1119
+ # This is the default interface from parent CSV
1120
+ default_interface_found = True
1121
+ existing_index = existing_interface.get('index')
1122
+
1123
+ # Check if we have new data for the same interface
1124
+ matching_new_interface = None
1125
+ for new_interface in new_interfaces:
1126
+ if (new_interface.get('index') == existing_index or
1127
+ new_interface.get('id') == existing_interface.get('id')):
1128
+ matching_new_interface = new_interface
1129
+ break
1130
+
1131
+ if matching_new_interface:
1132
+ # Merge the interfaces - keep default_lan=True from existing, but use any additional network ranges from new
1133
+ merged_interface = existing_interface.copy()
1134
+ merged_interface['network_ranges'] = matching_new_interface.get('network_ranges', [])
1135
+ # Preserve the virtual_interface flag from the new interface if present
1136
+ if matching_new_interface.get('virtual_interface'):
1137
+ merged_interface['virtual_interface'] = True
1138
+ final_interfaces.append(merged_interface)
1139
+ # Remove the matching interface from new_interfaces to avoid duplication
1140
+ new_interfaces.remove(matching_new_interface)
1141
+ else:
1142
+ # No new data for default interface, keep as-is
1143
+ final_interfaces.append(existing_interface)
1144
+ else:
1145
+ # Non-default existing interface, keep as-is
1146
+ final_interfaces.append(existing_interface)
1147
+
1148
+ # Add any remaining new interfaces that didn't match existing ones
1149
+ final_interfaces.extend(new_interfaces)
1150
+
1151
+ # Update site's lan_interfaces
1152
+ site['lan_interfaces'] = final_interfaces
1153
+
1154
+ except FileNotFoundError:
1155
+ print(f"Warning: Network ranges file '{ranges_csv_file}' not found for site {site['name']}")
1156
+ except Exception as e:
1157
+ print(f"Error loading network ranges from '{ranges_csv_file}': {e}")
1158
+ import traceback
1159
+ traceback.print_exc()
1160
+
1161
+
1162
+ def import_socket_sites_from_csv(args, configuration):
1163
+ """
1164
+ Main function to orchestrate the socket sites import process from CSV files
1165
+ """
1166
+ try:
1167
+ print(" Terraform Import Tool - Cato Socket Sites from CSV")
1168
+ print("=" * 70)
1169
+
1170
+ # Determine sites config directory
1171
+ sites_config_dir = None
1172
+ if hasattr(args, 'sites_config_dir') and args.sites_config_dir:
1173
+ sites_config_dir = args.sites_config_dir
1174
+ else:
1175
+ # Try to find sites_config directory relative to CSV file
1176
+ csv_dir = os.path.dirname(os.path.abspath(args.csv_file))
1177
+ potential_config_dir = os.path.join(csv_dir, 'sites_config')
1178
+ if os.path.exists(potential_config_dir):
1179
+ sites_config_dir = potential_config_dir
1180
+
1181
+ # Load data from CSV
1182
+ print(f" Loading data from {args.csv_file}...")
1183
+ if sites_config_dir:
1184
+ print(f" Loading network ranges from {sites_config_dir}...")
1185
+
1186
+ sites_data = load_csv_data(args.csv_file, sites_config_dir)
1187
+
1188
+ # Extract sites, WAN interfaces, LAN interfaces, and network ranges using existing function
1189
+ sites, wan_interfaces, lan_interfaces, network_ranges = extract_socket_sites_data(sites_data)
1190
+
1191
+ if hasattr(args, 'verbose') and args.verbose:
1192
+ print(f"\nExtracted data summary:")
1193
+ print(f" Sites: {len(sites)}")
1194
+ print(f" WAN Interfaces: {len(wan_interfaces)}")
1195
+ print(f" LAN Interfaces: {len(lan_interfaces)}")
1196
+ print(f" Network Ranges: {len(network_ranges)}")
1197
+
1198
+ print(f" Found {len(sites)} sites")
1199
+ print(f" Found {len(wan_interfaces)} WAN interfaces")
1200
+ print(f" Found {len(lan_interfaces)} LAN interfaces")
1201
+ print(f" Found {len(network_ranges)} network ranges")
1202
+
1203
+ if not sites and not wan_interfaces and not network_ranges:
1204
+ print(" No sites, interfaces, or network ranges found. Exiting.")
1205
+ return [{"success": False, "error": "No data found to import"}]
1206
+
1207
+ # Add module. prefix if not present
1208
+ module_name = args.module_name
1209
+ if not module_name.startswith('module.'):
1210
+ module_name = f'module.{module_name}'
1211
+
1212
+ # Generate Terraform configuration files if requested
1213
+ if hasattr(args, 'generate_only') and args.generate_only:
1214
+ print("\nGenerating Terraform configuration files...")
1215
+ output_dir = generate_terraform_import_files(sites, output_dir=getattr(args, 'output_dir', './imported_sites'))
1216
+ print(f"\nTerraform configuration files generated successfully in {output_dir}")
1217
+ print("\nNext steps:")
1218
+ print(f" 1. Copy the generated files to your Terraform project directory")
1219
+ print(f" 2. Run 'terraform init' to initialize")
1220
+ print(f" 3. Run 'terraform plan -generate-config-out=generated.tf' to generate configuration")
1221
+ print(f" 4. Run 'terraform apply' to import the resources")
1222
+
1223
+ return [{
1224
+ "success": True,
1225
+ "total_generated": len(sites),
1226
+ "output_dir": output_dir
1227
+ }]
1228
+
1229
+ # Validate Terraform environment before proceeding
1230
+ validate_terraform_environment(module_name, verbose=args.verbose)
1231
+
1232
+ # Ask for confirmation (unless auto-approved)
1233
+ # Determine which categories to import based on flags
1234
+ sites_only = getattr(args, 'sites_only', False)
1235
+ wan_only = getattr(args, 'wan_interfaces_only', False)
1236
+ lan_only = getattr(args, 'lan_interfaces_only', False)
1237
+ ranges_only = getattr(args, 'network_ranges_only', False)
1238
+
1239
+ import_summary = []
1240
+ if not (sites_only or wan_only or lan_only or ranges_only):
1241
+ import_summary.append(f"{len(sites)} sites")
1242
+ import_summary.append(f"{len(wan_interfaces)} WAN interfaces")
1243
+ import_summary.append(f"{len(lan_interfaces)} LAN interfaces")
1244
+ import_summary.append(f"{len(network_ranges)} network ranges")
1245
+ elif sites_only:
1246
+ import_summary.append(f"{len(sites)} sites only")
1247
+ elif wan_only:
1248
+ import_summary.append(f"{len(wan_interfaces)} WAN interfaces only")
1249
+ elif lan_only:
1250
+ import_summary.append(f"{len(lan_interfaces)} LAN interfaces only")
1251
+ elif ranges_only:
1252
+ import_summary.append(f"{len(network_ranges)} network ranges only")
1253
+
1254
+ print(f"\n Ready to import {', '.join(import_summary)}.")
1255
+
1256
+ if hasattr(args, 'auto_approve') and args.auto_approve:
1257
+ print("\nAuto-approve enabled, proceeding with import...")
1258
+ else:
1259
+ confirm = input(f"\nProceed with import? (y/n): ").lower()
1260
+ if confirm != 'y':
1261
+ print("Import cancelled.")
1262
+ return [{"success": False, "error": "Import cancelled by user"}]
1263
+
1264
+ total_successful = 0
1265
+ total_failed = 0
1266
+
1267
+ # Import sites first (if selected)
1268
+ if (sites_only or not (wan_only or lan_only or ranges_only)) and sites:
1269
+ successful, failed = import_socket_sites(sites, module_name=args.module_name,
1270
+ verbose=args.verbose, batch_size=getattr(args, 'batch_size', 10),
1271
+ delay_between_batches=getattr(args, 'delay', 2),
1272
+ auto_approve=getattr(args, 'auto_approve', False))
1273
+ total_successful += successful
1274
+ total_failed += failed
1275
+
1276
+ # Import WAN interfaces (if selected)
1277
+ if (wan_only or (not sites_only and not lan_only and not ranges_only)) and wan_interfaces:
1278
+ successful, failed = import_wan_interfaces(wan_interfaces, module_name=args.module_name,
1279
+ verbose=args.verbose, batch_size=getattr(args, 'batch_size', 10),
1280
+ delay_between_batches=getattr(args, 'delay', 2),
1281
+ auto_approve=getattr(args, 'auto_approve', False))
1282
+ total_successful += successful
1283
+ total_failed += failed
1284
+
1285
+ # Import LAN interfaces (if selected)
1286
+ if (lan_only or (not sites_only and not wan_only and not ranges_only)) and lan_interfaces:
1287
+ successful, failed = import_lan_interfaces(lan_interfaces, module_name=args.module_name,
1288
+ verbose=args.verbose, batch_size=getattr(args, 'batch_size', 10),
1289
+ delay_between_batches=getattr(args, 'delay', 2),
1290
+ auto_approve=getattr(args, 'auto_approve', False))
1291
+ total_successful += successful
1292
+ total_failed += failed
1293
+
1294
+ # Import network ranges (if selected)
1295
+ if (ranges_only or (not sites_only and not wan_only and not lan_only)) and network_ranges:
1296
+ successful, failed = import_network_ranges(network_ranges, lan_interfaces, module_name=args.module_name,
1297
+ verbose=args.verbose, batch_size=getattr(args, 'batch_size', 10),
1298
+ delay_between_batches=getattr(args, 'delay', 2),
1299
+ auto_approve=getattr(args, 'auto_approve', False))
1300
+ total_successful += successful
1301
+ total_failed += failed
1302
+
1303
+ # Final summary
1304
+ print("\n" + "=" * 70)
1305
+ print(" FINAL IMPORT SUMMARY")
1306
+ print("=" * 70)
1307
+ print(f" Total successful imports: {total_successful}")
1308
+ print(f" Total failed imports: {total_failed}")
1309
+ print(f" Overall success rate: {(total_successful / (total_successful + total_failed) * 100):.1f}%" if (total_successful + total_failed) > 0 else "N/A")
1310
+ print("\n Import process completed!")
1311
+
1312
+ return [{
1313
+ "success": True,
1314
+ "total_successful": total_successful,
1315
+ "total_failed": total_failed,
1316
+ "module_name": args.module_name
1317
+ }]
1318
+
1319
+ except KeyboardInterrupt:
1320
+ print("\nImport process cancelled by user (Ctrl+C).")
1321
+ print("Partial imports may have been completed.")
1322
+ return [{"success": False, "error": "Import cancelled by user"}]
1323
+ except Exception as e:
1324
+ print(f"ERROR: {str(e)}")
1325
+ return [{"success": False, "error": str(e)}]
1326
+
1327
+
1328
+ def convert_csv_to_json(args, configuration):
1329
+ """
1330
+ Convert CSV data to JSON format compatible with existing import tools
1331
+ """
1332
+ try:
1333
+ print(" CSV to JSON Converter - Cato Socket Sites")
1334
+ print("=" * 50)
1335
+
1336
+ # Determine sites config directory
1337
+ sites_config_dir = None
1338
+ if hasattr(args, 'sites_config_dir') and args.sites_config_dir:
1339
+ sites_config_dir = args.sites_config_dir
1340
+ else:
1341
+ # Try to find sites_config directory relative to CSV file
1342
+ csv_dir = os.path.dirname(os.path.abspath(args.csv_file))
1343
+ potential_config_dir = os.path.join(csv_dir, 'sites_config')
1344
+ if os.path.exists(potential_config_dir):
1345
+ sites_config_dir = potential_config_dir
1346
+
1347
+ # Load data from CSV
1348
+ print(f" Loading data from {args.csv_file}...")
1349
+ if sites_config_dir:
1350
+ print(f" Loading network ranges from {sites_config_dir}...")
1351
+
1352
+ sites_data = load_csv_data(args.csv_file, sites_config_dir)
1353
+
1354
+ # Create JSON structure
1355
+ json_data = {"sites": sites_data}
1356
+
1357
+ # Determine output filename
1358
+ if hasattr(args, 'output_file') and args.output_file:
1359
+ output_file = args.output_file
1360
+ else:
1361
+ # Generate output filename based on input CSV
1362
+ csv_base = os.path.splitext(os.path.basename(args.csv_file))[0]
1363
+ output_file = f"{csv_base}_converted.json"
1364
+
1365
+ # Write JSON file
1366
+ with open(output_file, 'w', encoding='utf-8') as f:
1367
+ json.dump(json_data, f, indent=2)
1368
+
1369
+ print(f" Converted {len(sites_data)} sites to JSON format")
1370
+ print(f" Output written to: {output_file}")
1371
+
1372
+ return [{
1373
+ "success": True,
1374
+ "input_file": args.csv_file,
1375
+ "output_file": output_file,
1376
+ "sites_count": len(sites_data)
1377
+ }]
1378
+
1379
+ except Exception as e:
1380
+ print(f"ERROR: {str(e)}")
1381
+ return [{"success": False, "error": str(e)}]