ms-cloud-endpoints 0.1.0__tar.gz

ms_cloud_endpoints-0.1.0/.gitignore

@@ -0,0 +1,12 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+build/
+dist/
+.venv/
+.pytest_cache/
+.ruff_cache/
+
+# Generated output
+output/

ms_cloud_endpoints-0.1.0/CHANGELOG.md

@@ -0,0 +1,27 @@
+# Changelog
+
+All notable changes to this project are documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to
+[Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+
+- `--ipv6`, the explicit counterpart of `--no-ipv6`. IPv6 is still included by default, and the last of the two
+  options wins.
+
+## [0.1.0]
+
+### Added
+
+- `ms-cloud-endpoints` command (also `python -m ms_cloud_endpoints`) that writes JSON and/or YAML.
+- Sources: Microsoft 365 IP Address and URL web service, the "other endpoints" Learn page (heuristic extraction),
+  Azure IP Ranges and Service Tags (download link discovered from the download page), Azure Private Endpoint
+  private DNS zone values, and Azure Backup geo-codes.
+- Public, China and US Government clouds (`--cloud`), source selection (`--source`), service tag filtering (`--tag`),
+  `--no-ipv6`, `--no-summary` and `--keep-going`.
+- Private Endpoint zone template expansion for `{regionName}`/`{region}` and `{regionCode}`.
+- Aggregated `summary` with deduplicated domains and collapsed IPv4/IPv6 prefixes.
+- A persisted per-machine `ClientRequestId` for the Microsoft 365 web service.

ms_cloud_endpoints-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Radux Technologies
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ms_cloud_endpoints-0.1.0/PKG-INFO

@@ -0,0 +1,281 @@
+Metadata-Version: 2.5
+Name: ms-cloud-endpoints
+Version: 0.1.0
+Summary: Collect Microsoft 365, Entra and Azure domains and IP ranges from the documented Microsoft sources and export them as JSON/YAML.
+Project-URL: Homepage, https://github.com/Jayonics/Azure-O365-DNS-UDR-SplitRouter
+Project-URL: Source, https://github.com/Jayonics/Azure-O365-DNS-UDR-SplitRouter.ms-cloud-endpoints
+Project-URL: Documentation, https://github.com/Jayonics/Azure-O365-DNS-UDR-SplitRouter.ms-cloud-endpoints/blob/main/README.md
+Project-URL: Issues, https://github.com/Jayonics/Azure-O365-DNS-UDR-SplitRouter.ms-cloud-endpoints/issues
+Project-URL: Changelog, https://github.com/Jayonics/Azure-O365-DNS-UDR-SplitRouter.ms-cloud-endpoints/blob/main/CHANGELOG.md
+Author-email: Jayonics <jameshopwood1998@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: azure,endpoints,entra,ip-ranges,microsoft-365,office-365,private-endpoint,service-tags,split-tunnel
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: System Administrators
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: System :: Networking
+Classifier: Typing :: Typed
+Requires-Python: >=3.12
+Requires-Dist: pyyaml>=6.0
+Description-Content-Type: text/markdown
+
+# ms-cloud-endpoints
+
+Collect the **domains and IP ranges of Microsoft 365, Microsoft Entra and Azure** from the sources Microsoft
+documents, and export them as one normalised **JSON and/or YAML** document.
+
+It is the data-collection stage of
+[Azure-O365-DNS-UDR-SplitRouter](https://github.com/Jayonics/Azure-O365-DNS-UDR-SplitRouter), which routes
+Microsoft traffic over a dedicated link with `systemd-resolved` routing domains and `systemd-networkd` routes. The
+output is also usable on its own, for example for firewall allow-lists, proxy PAC files or split-tunnel VPN
+profiles.
+
+- [Features](#features)
+- [Requirements](#requirements)
+- [Installation](#installation)
+- [Usage](#usage)
+- [Sources and clouds](#sources-and-clouds)
+- [Options](#options)
+- [Output schema](#output-schema)
+- [Notes and limitations](#notes-and-limitations)
+- [Development](#development)
+- [License](#license)
+
+## Features
+
+- Fetches all documented sources, finding the weekly Service Tag download link on the download page each run
+  instead of hard-coding it.
+- Public cloud by default, with China and US Government (Microsoft 365 GCC High/DoD) selectable.
+- Normalises everything: `snake_case` keys, lower-cased domains, IPv4 and IPv6 prefixes separated.
+- Expands Private Endpoint DNS zone templates (`{regionName}`, `{regionCode}`) into concrete zone names.
+- Optional `summary`: deduplicated domains, wildcard suffixes, root domains and collapsed IPv4/IPv6 prefix lists
+  across all sources.
+- Records the upstream versions (Microsoft 365 `version`, Service Tag `changeNumber`) so runs can be compared.
+- The only runtime dependency is [PyYAML](https://pypi.org/project/PyYAML/). HTTP and HTML parsing use the
+  standard library.
+
+## Requirements
+
+- Python 3.12 or later (tested on 3.12, 3.13 and 3.14)
+- Outbound HTTPS to `endpoints.office.com`, `www.microsoft.com`, `download.microsoft.com` and `learn.microsoft.com`
+
+## Installation
+
+From a clone of this repository (or the `ms-cloud-endpoints/` submodule of the
+[parent project](https://github.com/Jayonics/Azure-O365-DNS-UDR-SplitRouter)):
+
+```bash
+pipx install .        # isolated command-line install (recommended)
+# or
+pip install .         # into the current environment
+```
+
+Directly from GitHub:
+
+```bash
+pipx install "git+https://github.com/Jayonics/Azure-O365-DNS-UDR-SplitRouter.ms-cloud-endpoints.git"
+```
+
+Both install the `ms-cloud-endpoints` command. `python -m ms_cloud_endpoints` works too.
+
+For a version-pinned deployment once the release exists on PyPI:
+
+```bash
+pipx install "ms-cloud-endpoints==0.1.0"
+ms-cloud-endpoints --version
+# Upgrade or roll back by explicitly selecting the desired published version:
+pipx install --force "ms-cloud-endpoints==0.1.0"
+```
+
+Do not use `pipx upgrade`/`upgrade-all` for pinned deployments. Pins select the application version, not a full
+transitive dependency lock. For system-wide installation alongside the router, use `sudo pipx install --global`
+(pipx >= 1.5); see the parent's [deployment guide](https://github.com/Jayonics/Azure-O365-DNS-UDR-SplitRouter/blob/main/USAGE.md#version-pinned-deployment-from-pypi).
+
+## Usage
+
+```bash
+# Public cloud, all sources -> output/ms-cloud-endpoints.json and output/ms-cloud-endpoints.yaml
+ms-cloud-endpoints -o output/
+
+# Every cloud (Public, China, US Government)
+ms-cloud-endpoints --cloud all -o output/
+
+# Only the Entra ID service tags, IPv4 only, as YAML on stdout
+ms-cloud-endpoints --source service-tags --tag 'AzureActiveDirectory*' --no-ipv6 --format yaml
+
+# Microsoft 365 only, JSON to a file, keep going if a source fails
+ms-cloud-endpoints --source m365 --source m365-additional -o m365.json --keep-going
+```
+
+### Exit codes
+
+| Code | Meaning |
+|------|---------|
+| `0` | Success |
+| `1` | A source or writing the output failed (nothing is written) |
+| `2` | Usage error (invalid option or `ClientRequestId`) |
+| `3` | Partial output: one or more sources failed with `--keep-going` (failures are listed in `metadata.sources`) |
+
+## Sources and clouds
+
+`--cloud` selects which clouds to collect. It is repeatable, and `all` selects every cloud. The default is `public`.
+
+| Source (`--source`) | `public` | `china` | `usgov` |
+|---------------------|----------|---------|---------|
+| `m365`: [Microsoft 365 IP Address and URL web service](https://learn.microsoft.com/en-us/microsoft-365/enterprise/microsoft-365-ip-web-service) `version` + `endpoints` | `Worldwide` | `China` | `USGovDoD`, `USGovGCCHigh` |
+| `m365-additional`: [Other endpoints not included in the web service](https://learn.microsoft.com/en-us/microsoft-365/enterprise/additional-office365-ip-addresses-and-urls) | one page for every cloud. Rows are split by their cloud labels ("Public Cloud:", "GCC High, DoD …:", "…21Vianet:") | | |
+| `service-tags`: [Azure IP Ranges and Service Tags](https://www.microsoft.com/en-us/download/details.aspx?id=56519) | `Public` (id 56519) | `China` (id 57062) | `AzureGovernment` (id 57063) |
+| `private-dns`: [Private Endpoint DNS zone values](https://learn.microsoft.com/en-us/azure/private-link/private-endpoint-dns) + [Azure Backup geo-codes](https://learn.microsoft.com/en-us/azure/backup/scripts/geo-code-list) | `Commercial` | `China` | `Government` |
+
+Microsoft Entra is covered by the Microsoft 365 `Common` endpoint sets, the `AzureActiveDirectory*` service tags and
+the Entra rows on the "other endpoints" page.
+
+The Microsoft 365 `changes` method isn't used, because `endpoints` always returns a complete snapshot. Compare the
+recorded versions between runs instead.
+
+## Options
+
+| Option | Default | Meaning |
+|--------|---------|---------|
+| `--cloud public\|china\|usgov\|all` | `public` | Clouds to collect (repeatable) |
+| `--source m365\|m365-additional\|service-tags\|private-dns` | all | Sources to collect (repeatable) |
+| `--tag GLOB` | all tags | Only keep service tags whose name matches (case-insensitive glob, repeatable), e.g. `AzureCloud`, `Storage.*` |
+| `--ipv6`, `--no-ipv6` | `--ipv6` | Include IPv6 prefixes, or drop them everywhere. Recorded in `metadata.options.ipv6`. The last one given wins |
+| `--no-summary` | off | Omit the `summary` section |
+| `--format json\|yaml\|both` | `json` on stdout, `both` with `-o` | Output format |
+| `-o`, `--output PATH` | stdout | File, or directory (existing or ending in `/`) that receives `ms-cloud-endpoints.json`/`.yaml`. With `both` and a file path, the suffix is replaced per format |
+| `--client-request-id GUID` | see below | `ClientRequestId` sent to the Microsoft 365 web service |
+| `--keep-going` | off | Record failed sources in `metadata.sources` and still write the output |
+| `--timeout SECONDS` | `60` | HTTP timeout (three attempts with backoff on network errors, 429 and 5xx) |
+| `-v`, `-vv` | warnings | Info or debug logging on stderr |
+| `--version` | | Print the version |
+
+**ClientRequestId.** Microsoft asks for one GUID per machine. The value comes from `--client-request-id` first, then
+`$MS_ENDPOINTS_CLIENT_REQUEST_ID`. Otherwise a GUID is generated once and stored in
+`$XDG_CONFIG_HOME/ms-cloud-endpoints/client-request-id` (default `~/.config/…`). It's only created when the `m365`
+source is selected.
+
+## Output schema
+
+JSON and YAML have the same structure. Keys are `snake_case`, domains are lower-cased, and IP prefixes are
+normalised and split by family.
+
+```yaml
+metadata:            # generator, generator_version, generated_at, clouds, sources_selected, options, client_request_id
+  sources:           # one record per fetched document: source, scope, url, version, fetched_at, status[, error]
+m365:
+  Worldwide:
+    version: '2026081400'
+    endpoint_sets:   # id, service_area, service_area_display_name, category, required, express_route,
+                     # urls, ipv4, ipv6, tcp_ports, udp_ports, notes
+m365_additional:
+  source_url: …
+  rows:              # row, purpose, source, destination_text (raw), type, domains, ipv4, ipv6,
+                     # by_cloud.{public,usgov,china}.{domains,ipv4,ipv6}
+azure_service_tags:
+  Public:            # cloud, change_number, regions, source_url
+    tags:            # name, id, region, region_id, system_service, platform, network_features, change_number, ipv4, ipv6
+private_dns_zones:
+  backup_geo_codes:  # geo_code, region_name, region
+  Commercial:        # category, resource_type, subresources, zone_templates, zones, forwarder_templates,
+                     # forwarders, unresolved_placeholders, footnotes
+summary:
+  domains:           # exact, wildcard, wildcard_suffixes, root_domains
+  ipv4: []           # collapsed across all selected sources and clouds
+  ipv6: []
+  counts: {}
+```
+
+## Notes and limitations
+
+- **Private Endpoint templates.**
+  - `{regionName}` and `{region}` are expanded with the regions from the same cloud's service tags, so they stay
+    unexpanded unless `service-tags` is collected in the same run.
+  - `{regionCode}` is expanded with the lower-cased Azure Backup geo-codes.
+  - Other placeholders (`{dnsPrefix}`, `{subzone}`, `{partitionId}`, …) can't be expanded and are listed in
+    `unresolved_placeholders`.
+  - In the summary every placeholder counts as a `*` wildcard, so `{regionName}.kusto.windows.net` adds the suffix
+    `kusto.windows.net`.
+- **"Other endpoints" page.** This page is free text, so extraction is heuristic: FQDNs, wildcards, IPs and CIDRs
+  are pulled out of the prose. The raw `destination_text` is always kept. When the destination is empty, the
+  purpose text is scanned instead.
+- **Third-party domains.** The summary includes the certificate authority and CDN domains Microsoft lists (for
+  example `digicert.com`, `akamaihd.net`). Filter them out if you use the list for DNS routing. `root_domains` takes
+  the last two labels (three for known multi-label suffixes such as `com.cn`). It doesn't use the Public Suffix
+  List.
+- **Upstream HTML changes.** The Learn pages are parsed by section id and table header text. If Microsoft changes
+  their structure, the affected source fails with a clear error instead of returning partial data.
+- **Size.** All Public service tags make the JSON about 6 MB. Use `--tag` to narrow it.
+- **Refresh cadence.** Microsoft 365 endpoints change monthly (with 30 days' notice). Service tags are published
+  weekly. Check the Microsoft 365 `version` at most once an hour.
+
+## Development
+
+Development dependencies are declared as a [PEP 735](https://peps.python.org/pep-0735/) dependency group (`dev`:
+pytest and ruff).
+
+```bash
+python -m venv .venv && . .venv/bin/activate
+pip install -e . --group dev      # pip >= 25.1; with uv: `uv sync`
+
+pytest                            # offline tests against the fixtures in tests/fixtures/
+ruff check . && ruff format --check .
+python -m build                   # sdist + wheel in dist/ (pip install build)
+```
+
+Layout:
+
+```
+src/ms_cloud_endpoints/
+  cli.py            command line, orchestration, exit codes
+  clouds.py         cloud name -> upstream identifiers
+  fetch.py          urllib client with retries (Fetcher protocol)
+  html_tables.py    html.parser DOM + Learn page section/table helpers
+  netutil.py        prefix/domain normalisation and extraction
+  output.py         JSON/YAML writers (atomic file writes)
+  summary.py        cross-source aggregation
+  sources/          one module per upstream source
+tests/              pytest suite; fixtures/ holds trimmed copies of each upstream document
+```
+
+The repository's [pre-commit configuration](.pre-commit-config.yaml)
+runs ruff on this project (`pre-commit install`). CI runs lint, the tests on Python 3.12 to 3.14, and a package
+build.
+
+### Releases
+
+The existing workflow also runs on `v*` tag pushes. Stable tags such as `v0.1.0` must exactly match the version
+in the built wheel (sourced from `src/ms_cloud_endpoints/__init__.py`); prerelease or mismatched tags fail.
+After lint, tests and build checks pass, the `pypi` environment gates PyPI Trusted Publishing. A subsequent
+job creates a GitHub Release with the same wheel and source distribution. Pull requests, branch pushes and
+manual runs never publish.
+
+Before tagging, configure this repository's `pypi` GitHub environment to allow deployments only from `v*`
+tags (optionally with required reviewers, where the GitHub plan supports them). Register a PyPI Trusted Publisher for project `ms-cloud-endpoints`,
+owner `Jayonics`, repository `Azure-O365-DNS-UDR-SplitRouter.ms-cloud-endpoints`, workflow
+`ms-cloud-endpoints.yml`, environment `pypi`. No PyPI API token is required. Optionally restrict release-tag
+creation and mutation with a tag ruleset.
+
+Commit the version and changelog first, then release the reviewed commit:
+
+```bash
+git tag -a v0.1.0 -m "Release 0.1.0"
+git push origin main
+git push origin v0.1.0
+```
+
+Pushing the tag publishes automatically once checks pass (or after approval, if reviewers are required). Never move a published tag or reuse an uploaded PyPI
+version. If GitHub release creation fails after PyPI succeeds, re-run only failed jobs, not the whole workflow.
+See the parent's [release procedure](https://github.com/Jayonics/Azure-O365-DNS-UDR-SplitRouter/blob/main/USAGE.md#publishing-releases)
+for the complete setup and coordinated deployment process.
+
+## License
+
+[MIT](LICENSE) © Radux Technologies

ms_cloud_endpoints-0.1.0/README.md

@@ -0,0 +1,253 @@
+# ms-cloud-endpoints
+
+Collect the **domains and IP ranges of Microsoft 365, Microsoft Entra and Azure** from the sources Microsoft
+documents, and export them as one normalised **JSON and/or YAML** document.
+
+It is the data-collection stage of
+[Azure-O365-DNS-UDR-SplitRouter](https://github.com/Jayonics/Azure-O365-DNS-UDR-SplitRouter), which routes
+Microsoft traffic over a dedicated link with `systemd-resolved` routing domains and `systemd-networkd` routes. The
+output is also usable on its own, for example for firewall allow-lists, proxy PAC files or split-tunnel VPN
+profiles.
+
+- [Features](#features)
+- [Requirements](#requirements)
+- [Installation](#installation)
+- [Usage](#usage)
+- [Sources and clouds](#sources-and-clouds)
+- [Options](#options)
+- [Output schema](#output-schema)
+- [Notes and limitations](#notes-and-limitations)
+- [Development](#development)
+- [License](#license)
+
+## Features
+
+- Fetches all documented sources, finding the weekly Service Tag download link on the download page each run
+  instead of hard-coding it.
+- Public cloud by default, with China and US Government (Microsoft 365 GCC High/DoD) selectable.
+- Normalises everything: `snake_case` keys, lower-cased domains, IPv4 and IPv6 prefixes separated.
+- Expands Private Endpoint DNS zone templates (`{regionName}`, `{regionCode}`) into concrete zone names.
+- Optional `summary`: deduplicated domains, wildcard suffixes, root domains and collapsed IPv4/IPv6 prefix lists
+  across all sources.
+- Records the upstream versions (Microsoft 365 `version`, Service Tag `changeNumber`) so runs can be compared.
+- The only runtime dependency is [PyYAML](https://pypi.org/project/PyYAML/). HTTP and HTML parsing use the
+  standard library.
+
+## Requirements
+
+- Python 3.12 or later (tested on 3.12, 3.13 and 3.14)
+- Outbound HTTPS to `endpoints.office.com`, `www.microsoft.com`, `download.microsoft.com` and `learn.microsoft.com`
+
+## Installation
+
+From a clone of this repository (or the `ms-cloud-endpoints/` submodule of the
+[parent project](https://github.com/Jayonics/Azure-O365-DNS-UDR-SplitRouter)):
+
+```bash
+pipx install .        # isolated command-line install (recommended)
+# or
+pip install .         # into the current environment
+```
+
+Directly from GitHub:
+
+```bash
+pipx install "git+https://github.com/Jayonics/Azure-O365-DNS-UDR-SplitRouter.ms-cloud-endpoints.git"
+```
+
+Both install the `ms-cloud-endpoints` command. `python -m ms_cloud_endpoints` works too.
+
+For a version-pinned deployment once the release exists on PyPI:
+
+```bash
+pipx install "ms-cloud-endpoints==0.1.0"
+ms-cloud-endpoints --version
+# Upgrade or roll back by explicitly selecting the desired published version:
+pipx install --force "ms-cloud-endpoints==0.1.0"
+```
+
+Do not use `pipx upgrade`/`upgrade-all` for pinned deployments. Pins select the application version, not a full
+transitive dependency lock. For system-wide installation alongside the router, use `sudo pipx install --global`
+(pipx >= 1.5); see the parent's [deployment guide](https://github.com/Jayonics/Azure-O365-DNS-UDR-SplitRouter/blob/main/USAGE.md#version-pinned-deployment-from-pypi).
+
+## Usage
+
+```bash
+# Public cloud, all sources -> output/ms-cloud-endpoints.json and output/ms-cloud-endpoints.yaml
+ms-cloud-endpoints -o output/
+
+# Every cloud (Public, China, US Government)
+ms-cloud-endpoints --cloud all -o output/
+
+# Only the Entra ID service tags, IPv4 only, as YAML on stdout
+ms-cloud-endpoints --source service-tags --tag 'AzureActiveDirectory*' --no-ipv6 --format yaml
+
+# Microsoft 365 only, JSON to a file, keep going if a source fails
+ms-cloud-endpoints --source m365 --source m365-additional -o m365.json --keep-going
+```
+
+### Exit codes
+
+| Code | Meaning |
+|------|---------|
+| `0` | Success |
+| `1` | A source or writing the output failed (nothing is written) |
+| `2` | Usage error (invalid option or `ClientRequestId`) |
+| `3` | Partial output: one or more sources failed with `--keep-going` (failures are listed in `metadata.sources`) |
+
+## Sources and clouds
+
+`--cloud` selects which clouds to collect. It is repeatable, and `all` selects every cloud. The default is `public`.
+
+| Source (`--source`) | `public` | `china` | `usgov` |
+|---------------------|----------|---------|---------|
+| `m365`: [Microsoft 365 IP Address and URL web service](https://learn.microsoft.com/en-us/microsoft-365/enterprise/microsoft-365-ip-web-service) `version` + `endpoints` | `Worldwide` | `China` | `USGovDoD`, `USGovGCCHigh` |
+| `m365-additional`: [Other endpoints not included in the web service](https://learn.microsoft.com/en-us/microsoft-365/enterprise/additional-office365-ip-addresses-and-urls) | one page for every cloud. Rows are split by their cloud labels ("Public Cloud:", "GCC High, DoD …:", "…21Vianet:") | | |
+| `service-tags`: [Azure IP Ranges and Service Tags](https://www.microsoft.com/en-us/download/details.aspx?id=56519) | `Public` (id 56519) | `China` (id 57062) | `AzureGovernment` (id 57063) |
+| `private-dns`: [Private Endpoint DNS zone values](https://learn.microsoft.com/en-us/azure/private-link/private-endpoint-dns) + [Azure Backup geo-codes](https://learn.microsoft.com/en-us/azure/backup/scripts/geo-code-list) | `Commercial` | `China` | `Government` |
+
+Microsoft Entra is covered by the Microsoft 365 `Common` endpoint sets, the `AzureActiveDirectory*` service tags and
+the Entra rows on the "other endpoints" page.
+
+The Microsoft 365 `changes` method isn't used, because `endpoints` always returns a complete snapshot. Compare the
+recorded versions between runs instead.
+
+## Options
+
+| Option | Default | Meaning |
+|--------|---------|---------|
+| `--cloud public\|china\|usgov\|all` | `public` | Clouds to collect (repeatable) |
+| `--source m365\|m365-additional\|service-tags\|private-dns` | all | Sources to collect (repeatable) |
+| `--tag GLOB` | all tags | Only keep service tags whose name matches (case-insensitive glob, repeatable), e.g. `AzureCloud`, `Storage.*` |
+| `--ipv6`, `--no-ipv6` | `--ipv6` | Include IPv6 prefixes, or drop them everywhere. Recorded in `metadata.options.ipv6`. The last one given wins |
+| `--no-summary` | off | Omit the `summary` section |
+| `--format json\|yaml\|both` | `json` on stdout, `both` with `-o` | Output format |
+| `-o`, `--output PATH` | stdout | File, or directory (existing or ending in `/`) that receives `ms-cloud-endpoints.json`/`.yaml`. With `both` and a file path, the suffix is replaced per format |
+| `--client-request-id GUID` | see below | `ClientRequestId` sent to the Microsoft 365 web service |
+| `--keep-going` | off | Record failed sources in `metadata.sources` and still write the output |
+| `--timeout SECONDS` | `60` | HTTP timeout (three attempts with backoff on network errors, 429 and 5xx) |
+| `-v`, `-vv` | warnings | Info or debug logging on stderr |
+| `--version` | | Print the version |
+
+**ClientRequestId.** Microsoft asks for one GUID per machine. The value comes from `--client-request-id` first, then
+`$MS_ENDPOINTS_CLIENT_REQUEST_ID`. Otherwise a GUID is generated once and stored in
+`$XDG_CONFIG_HOME/ms-cloud-endpoints/client-request-id` (default `~/.config/…`). It's only created when the `m365`
+source is selected.
+
+## Output schema
+
+JSON and YAML have the same structure. Keys are `snake_case`, domains are lower-cased, and IP prefixes are
+normalised and split by family.
+
+```yaml
+metadata:            # generator, generator_version, generated_at, clouds, sources_selected, options, client_request_id
+  sources:           # one record per fetched document: source, scope, url, version, fetched_at, status[, error]
+m365:
+  Worldwide:
+    version: '2026081400'
+    endpoint_sets:   # id, service_area, service_area_display_name, category, required, express_route,
+                     # urls, ipv4, ipv6, tcp_ports, udp_ports, notes
+m365_additional:
+  source_url: …
+  rows:              # row, purpose, source, destination_text (raw), type, domains, ipv4, ipv6,
+                     # by_cloud.{public,usgov,china}.{domains,ipv4,ipv6}
+azure_service_tags:
+  Public:            # cloud, change_number, regions, source_url
+    tags:            # name, id, region, region_id, system_service, platform, network_features, change_number, ipv4, ipv6
+private_dns_zones:
+  backup_geo_codes:  # geo_code, region_name, region
+  Commercial:        # category, resource_type, subresources, zone_templates, zones, forwarder_templates,
+                     # forwarders, unresolved_placeholders, footnotes
+summary:
+  domains:           # exact, wildcard, wildcard_suffixes, root_domains
+  ipv4: []           # collapsed across all selected sources and clouds
+  ipv6: []
+  counts: {}
+```
+
+## Notes and limitations
+
+- **Private Endpoint templates.**
+  - `{regionName}` and `{region}` are expanded with the regions from the same cloud's service tags, so they stay
+    unexpanded unless `service-tags` is collected in the same run.
+  - `{regionCode}` is expanded with the lower-cased Azure Backup geo-codes.
+  - Other placeholders (`{dnsPrefix}`, `{subzone}`, `{partitionId}`, …) can't be expanded and are listed in
+    `unresolved_placeholders`.
+  - In the summary every placeholder counts as a `*` wildcard, so `{regionName}.kusto.windows.net` adds the suffix
+    `kusto.windows.net`.
+- **"Other endpoints" page.** This page is free text, so extraction is heuristic: FQDNs, wildcards, IPs and CIDRs
+  are pulled out of the prose. The raw `destination_text` is always kept. When the destination is empty, the
+  purpose text is scanned instead.
+- **Third-party domains.** The summary includes the certificate authority and CDN domains Microsoft lists (for
+  example `digicert.com`, `akamaihd.net`). Filter them out if you use the list for DNS routing. `root_domains` takes
+  the last two labels (three for known multi-label suffixes such as `com.cn`). It doesn't use the Public Suffix
+  List.
+- **Upstream HTML changes.** The Learn pages are parsed by section id and table header text. If Microsoft changes
+  their structure, the affected source fails with a clear error instead of returning partial data.
+- **Size.** All Public service tags make the JSON about 6 MB. Use `--tag` to narrow it.
+- **Refresh cadence.** Microsoft 365 endpoints change monthly (with 30 days' notice). Service tags are published
+  weekly. Check the Microsoft 365 `version` at most once an hour.
+
+## Development
+
+Development dependencies are declared as a [PEP 735](https://peps.python.org/pep-0735/) dependency group (`dev`:
+pytest and ruff).
+
+```bash
+python -m venv .venv && . .venv/bin/activate
+pip install -e . --group dev      # pip >= 25.1; with uv: `uv sync`
+
+pytest                            # offline tests against the fixtures in tests/fixtures/
+ruff check . && ruff format --check .
+python -m build                   # sdist + wheel in dist/ (pip install build)
+```
+
+Layout:
+
+```
+src/ms_cloud_endpoints/
+  cli.py            command line, orchestration, exit codes
+  clouds.py         cloud name -> upstream identifiers
+  fetch.py          urllib client with retries (Fetcher protocol)
+  html_tables.py    html.parser DOM + Learn page section/table helpers
+  netutil.py        prefix/domain normalisation and extraction
+  output.py         JSON/YAML writers (atomic file writes)
+  summary.py        cross-source aggregation
+  sources/          one module per upstream source
+tests/              pytest suite; fixtures/ holds trimmed copies of each upstream document
+```
+
+The repository's [pre-commit configuration](.pre-commit-config.yaml)
+runs ruff on this project (`pre-commit install`). CI runs lint, the tests on Python 3.12 to 3.14, and a package
+build.
+
+### Releases
+
+The existing workflow also runs on `v*` tag pushes. Stable tags such as `v0.1.0` must exactly match the version
+in the built wheel (sourced from `src/ms_cloud_endpoints/__init__.py`); prerelease or mismatched tags fail.
+After lint, tests and build checks pass, the `pypi` environment gates PyPI Trusted Publishing. A subsequent
+job creates a GitHub Release with the same wheel and source distribution. Pull requests, branch pushes and
+manual runs never publish.
+
+Before tagging, configure this repository's `pypi` GitHub environment to allow deployments only from `v*`
+tags (optionally with required reviewers, where the GitHub plan supports them). Register a PyPI Trusted Publisher for project `ms-cloud-endpoints`,
+owner `Jayonics`, repository `Azure-O365-DNS-UDR-SplitRouter.ms-cloud-endpoints`, workflow
+`ms-cloud-endpoints.yml`, environment `pypi`. No PyPI API token is required. Optionally restrict release-tag
+creation and mutation with a tag ruleset.
+
+Commit the version and changelog first, then release the reviewed commit:
+
+```bash
+git tag -a v0.1.0 -m "Release 0.1.0"
+git push origin main
+git push origin v0.1.0
+```
+
+Pushing the tag publishes automatically once checks pass (or after approval, if reviewers are required). Never move a published tag or reuse an uploaded PyPI
+version. If GitHub release creation fails after PyPI succeeds, re-run only failed jobs, not the whole workflow.
+See the parent's [release procedure](https://github.com/Jayonics/Azure-O365-DNS-UDR-SplitRouter/blob/main/USAGE.md#publishing-releases)
+for the complete setup and coordinated deployment process.
+
+## License
+
+[MIT](LICENSE) © Radux Technologies

ms_cloud_endpoints-0.1.0/pyproject.toml

@@ -0,0 +1,75 @@
+[build-system]
+requires = ["hatchling>=1.27"]
+build-backend = "hatchling.build"
+
+[project]
+name = "ms-cloud-endpoints"
+dynamic = ["version"]
+description = "Collect Microsoft 365, Entra and Azure domains and IP ranges from the documented Microsoft sources and export them as JSON/YAML."
+readme = "README.md"
+requires-python = ">=3.12"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Jayonics", email = "jameshopwood1998@gmail.com" }]
+keywords = [
+    "microsoft-365",
+    "office-365",
+    "azure",
+    "entra",
+    "service-tags",
+    "ip-ranges",
+    "endpoints",
+    "split-tunnel",
+    "private-endpoint",
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Environment :: Console",
+    "Intended Audience :: System Administrators",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Topic :: System :: Networking",
+    "Typing :: Typed",
+]
+dependencies = ["PyYAML>=6.0"]
+
+[project.urls]
+Homepage = "https://github.com/Jayonics/Azure-O365-DNS-UDR-SplitRouter"
+Source = "https://github.com/Jayonics/Azure-O365-DNS-UDR-SplitRouter.ms-cloud-endpoints"
+Documentation = "https://github.com/Jayonics/Azure-O365-DNS-UDR-SplitRouter.ms-cloud-endpoints/blob/main/README.md"
+Issues = "https://github.com/Jayonics/Azure-O365-DNS-UDR-SplitRouter.ms-cloud-endpoints/issues"
+Changelog = "https://github.com/Jayonics/Azure-O365-DNS-UDR-SplitRouter.ms-cloud-endpoints/blob/main/CHANGELOG.md"
+
+[project.scripts]
+ms-cloud-endpoints = "ms_cloud_endpoints.cli:main"
+
+[dependency-groups]
+dev = ["pytest>=8", "ruff>=0.6"]
+
+[tool.hatch.version]
+path = "src/ms_cloud_endpoints/__init__.py"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/ms_cloud_endpoints"]
+
+[tool.hatch.build.targets.sdist]
+include = ["src/", "tests/", "README.md", "CHANGELOG.md", "LICENSE"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = ["--import-mode=importlib"]
+filterwarnings = ["error"]
+
+[tool.ruff]
+line-length = 120
+target-version = "py312"
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "I", "UP", "B"]
+
+[tool.ruff.lint.isort]
+known-first-party = ["ms_cloud_endpoints"]