polyswarm-api 4.4.0__tar.gz → 4.7.0__tar.gz

{polyswarm_api-4.4.0/src/polyswarm_api.egg-info → polyswarm_api-4.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polyswarm_api
-Version: 4.4.0
+Version: 4.7.0
 Summary: Client library to simplify interacting with the PolySwarm consumer API
 Author-email: PolySwarm Developers <info@polyswarm.io>
 License: MIT

{polyswarm_api-4.4.0 → polyswarm_api-4.7.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "polyswarm_api"
-version = "4.4.0"
+version = "4.7.0"
 description = "Client library to simplify interacting with the PolySwarm consumer API"
 readme = "README.md"
 requires-python = ">=3.10,<4"
@@ -55,7 +55,7 @@ package-dir = { "" = "src" }
 where = ["src"]
 
 [tool.bumpversion]
-current_version = "4.4.0"
+current_version = "4.7.0"
 commit = true
 tag = false
 sign_tags = true

{polyswarm_api-4.4.0 → polyswarm_api-4.7.0}/src/polyswarm_api/__init__.py

@@ -1,9 +1,10 @@
 # https://www.python.org/dev/peps/pep-0008/#module-level-dunder-names
-__version__ = '4.4.0'
+__version__ = '4.7.0'
 __release_url__ = 'https://api.github.com/repos/polyswarm/polyswarm-api/releases/latest'
 
 from . import api
 from . import exceptions
+from . import refang
 from .api import PolyswarmAPI
 from .session import PolyswarmSession
 from .aio import PolySwarmAsyncAPI, AsyncPolyswarmSession

{polyswarm_api-4.4.0 → polyswarm_api-4.7.0}/src/polyswarm_api/aio/api.py

@@ -17,7 +17,7 @@ import io
 import logging
 import time
 
-from polyswarm_api import exceptions, resources, settings
+from polyswarm_api import exceptions, refang, resources, settings
 from polyswarm_api.core import PolyswarmRequest, _as_result_bound
 
 from .session import AsyncPolyswarmSession
@@ -53,6 +53,7 @@ class PolySwarmAsyncAPI:
         verify: bool = True,
         *,
         session: AsyncPolyswarmSession | None = None,
+        refang_iocs: bool = False,
         **httpx_kwargs,
     ):
         key_masked = '******' + (key[-4:] if key and len(key) > 16 else '')
@@ -64,6 +65,11 @@ class PolySwarmAsyncAPI:
         self.community = community or settings.DEFAULT_COMMUNITY
         self.timeout = timeout or settings.DEFAULT_HTTP_TIMEOUT
         self.verify = verify
+        # Refang defanged URL / domain / IP inputs (``hxxps[:]//evil[.]com``)
+        # before building a request. Opt-in: the default preserves the 4.5.0
+        # behaviour of sending every input verbatim. See ``polyswarm_api.refang``
+        # and ``_refang`` below for exactly which inputs are touched.
+        self.refang_iocs = refang_iocs
         self._engines = None
         # Either accept a pre-built session (customization point) or
         # build the default from ``key``. Passing both is ambiguous.
@@ -88,6 +94,27 @@ class PolySwarmAsyncAPI:
         attrs = f'uri={self.uri!r}, community={self.community!r}, timeout={self.timeout!r}'
         return f'<{clsname}({attrs}) at 0x{id(self):x}>'
 
+    def _refang(self, value):
+        """Refang one URL / domain / IP input when ``refang_iocs`` is on.
+
+        Applied only to arguments that name a network indicator (a URL to
+        search or submit, an ``ips=`` / ``urls=`` / ``domains=`` entry, an IoC
+        ``ip`` / ``domain``, a known-host ``host``) — never to a free-form
+        metadata query or a hash.
+        Anything that is not a defanged network IoC is returned unchanged.
+        """
+        if not self.refang_iocs:
+            return value
+        return refang.refang_ioc(value)
+
+    def _refang_all(self, values):
+        """``_refang`` over a list argument; ``None`` / empty pass through."""
+        if not values or not self.refang_iocs:
+            return values
+        if isinstance(values, str):
+            return self._refang(values)
+        return [self._refang(v) for v in values]
+
     async def aclose(self):
         """Close the underlying HTTP client."""
         await self.session.aclose()
@@ -340,6 +367,7 @@ class PolySwarmAsyncAPI:
         :param url: A url to be searched by exact match
         :return: Generator of ArtifactInstance resources
         """
+        url = self._refang(url)
         logger.info('Searching for url %s', url)
         async for item in self._paginate(resources.ArtifactInstance.search_url(self, url)):
             yield item
@@ -365,6 +393,7 @@ class PolySwarmAsyncAPI:
         :param exclude: A list of fields to be excluded from the result (.* wildcards are accepted)
         :return: Generator of ArtifactInstance resources
         """
+        ips, urls, domains = self._refang_all(ips), self._refang_all(urls), self._refang_all(domains)
         logger.info('Searching for metadata %s', query)
         async for item in self._paginate(resources.Metadata.get(self, query=query, community=self.community, include=include, exclude=exclude, ips=ips, urls=urls, domains=domains)):
             yield item
@@ -381,18 +410,28 @@ class PolySwarmAsyncAPI:
         async for item in self._paginate(resources.IOC.iocs_by_hash(self, hash_value, hash_type, hide_known_good=hide_known_good, beta=beta)):
             yield item
 
-    async def search_by_ioc(self, ip=None, domain=None, ttp=None, imphash=None):
+    async def search_by_ioc(self, ip=None, domain=None, ttp=None, imphash=None, with_artifacts=False):
         """
         Search artifacts by IOC (ip, domain, ttp, or imphash)
-        
+
         :param ip: ip address to search by
         :param domain: domain address to search by
         :param ttp: ttp to search by
         :param imphash: ImpHash to search by
-        :return: Generator of ArtifactInstance resources
-        """
-        logger.info('Searching by ioc %s', dict(ip=ip, domain=domain, ttp=ttp, imphash=imphash))
-        async for item in self._paginate(resources.IOC.ioc_search(self, ip=ip, domain=domain, ttp=ttp, imphash=imphash)):
+        :param with_artifacts: True yields a Metadata resource per matching artifact
+            (a metadata-search row trimmed to a field set the server owns: artifact.*,
+            the scan summary, ssdeep/tlsh and the malware family) instead of its bare
+            sha256. With no ip/domain/ttp/imphash the server refuses it with a 400
+            (a typed exception); without it a bare call behaves as it always has.
+            Needs a server that supports the parameter:
+            an older one ignores it and answers bare sha256 strings, which fail to
+            parse as Metadata (TypeError).
+        :return: Generator of IOC resources whose ``json`` is a sha256 string, or of
+            Metadata resources when ``with_artifacts`` is True
+        """
+        ip, domain = self._refang(ip), self._refang(domain)
+        logger.info('Searching by ioc %s', dict(ip=ip, domain=domain, ttp=ttp, imphash=imphash, with_artifacts=with_artifacts))
+        async for item in self._paginate(resources.IOC.ioc_search(self, ip=ip, domain=domain, ttp=ttp, imphash=imphash, with_artifacts=with_artifacts)):
             yield item
 
     async def check_known_hosts(self, ips=[], domains=[]):
@@ -403,6 +442,7 @@ class PolySwarmAsyncAPI:
         :param domains
         :return: Generator of IOC resources
         """
+        ips, domains = self._refang_all(ips), self._refang_all(domains)
         logger.info('Checking known hosts ips: %s, domains: %s', ips, domains)
         async for item in self._paginate(resources.IOC.check_known_hosts(self, ips, domains)):
             yield item
@@ -416,6 +456,7 @@ class PolySwarmAsyncAPI:
         :param host
         :return: IOC resource
         """
+        host = self._refang(host)
         logger.info('Creating known good ioc %s %s %s', type, host, source)
         return await self._single(resources.IOC.create_known_good(self, type, host, source))
 
@@ -428,6 +469,7 @@ class PolySwarmAsyncAPI:
         :param host
         :return: IOC resource
         """
+        host = self._refang(host)
         logger.info('Creating known bad ioc %s %s %s', type, host, source)
         return await self._single(resources.IOC.create_known_bad(self, type, host, source))
 
@@ -440,6 +482,7 @@ class PolySwarmAsyncAPI:
         :param host
         :return: IOC resource
         """
+        host = self._refang(host)
         logger.info('Updating known good ioc %s %s %s %s', id, type, host, source)
         return await self._single(resources.IOC.update_known_good(self, id, type, host, source, good))
 
@@ -697,7 +740,8 @@ class PolySwarmAsyncAPI:
         return await self._single(resources.YaraRuleset.delete(self, id=ruleset_id, community=self.community))
 
     async def ruleset_list(self, name=None, status=None, favorites_only=None,
-                           has_new_results=None):
+                           has_new_results=None, sort=None,
+                           exclude_favorites=None):
         """
         List all YaraRulesets for the current account.
 
@@ -706,17 +750,53 @@ class PolySwarmAsyncAPI:
         :param status: 'active' returns only rulesets whose live hunt is
             currently running.
         :param favorites_only: True returns only favorited rulesets.
+        :param exclude_favorites: True returns only the rulesets that are NOT
+            favorited — the inverse of ``favorites_only``, and refused together
+            with it (a contradiction, answered with an error rather than an
+            empty list). It exists for clients that render the favorites as
+            their own list: the favorites are a separate, unpaginated fetch
+            bounded by the account's budget, so leaving them in the paginated
+            list too makes a page either repeat a row or come back short.
+            Appended to the signature rather than placed beside
+            ``favorites_only`` so a positional caller keeps working.
         :param has_new_results: True returns only rulesets whose stored
             new-results counter is positive. The counter (and its window) is
             maintained server-side by a scheduled refresh; rows carry it as
             ``new_results_count`` with ``new_results_counted_at`` marking when
             it was last refreshed. There is no per-request window parameter.
+        :param sort: ``'active_first'`` returns the rulesets that carry a live
+            hunt link first, newest first within each block. Default (None) is
+            newest first. "Newest first" is the server's own insertion key, NOT
+            the ``id`` on the rows you get back — that one is unique but
+            unordered, so dedupe with it and never resume or bound a walk with
+            it. Applied SERVER-side, across pages — the list is
+            keyset-paginated, so a client-side sort would only ever reorder one
+            page; the SDK never re-orders rows. Reuse a page's ``offset`` only
+            with the same ``sort``: the server refuses a cursor minted under
+            the other order.
+
+            Two server-side properties of that key, neither of them SDK
+            behaviour. It ranks on the stored link, which is a WIDER predicate
+            than the one ``livescan_id`` is rendered under: a legacy row whose
+            hunt was stopped without clearing the link ranks in the leading
+            block while still serializing ``livescan_id`` as ``None``. Read the
+            field to decide whether a ruleset is running; never the position.
+
+            And the key is MUTABLE, unlike that default: a ruleset whose
+            live hunt stops mid-walk falls back into the idle block below the
+            cursor and is yielded twice, and one started mid-walk moves above
+            the cursor and is skipped for the rest of that walk. That is a
+            property of the walk, so starting fresh from the first page does
+            not avoid it. This generator streams pages and does not dedupe —
+            dedupe by ``id`` if you consume more than one page.
         :return: A generator of YaraRuleset resources
         """
         logger.info('List rulesets')
         async for item in self._paginate(resources.YaraRuleset.list(
                 self, name=name, status=status, favorites_only=favorites_only,
-                has_new_results=has_new_results, community=self.community)):
+                has_new_results=has_new_results, sort=sort,
+                exclude_favorites=exclude_favorites,
+                community=self.community)):
             yield item
 
     async def ruleset_favorite(self, ruleset_id, favorite=True):
@@ -1336,11 +1416,12 @@ class PolySwarmAsyncAPI:
                     self, artifact, artifact_type=artifact_type, artifact_name=artifact_name
                 )
             elif artifact_type == resources.ArtifactType.URL:
-                if preprocessing and preprocessing["type"] == "qrcode":
+                if preprocessing and preprocessing.get("type") == "qrcode":
                     artifact = resources.LocalArtifact.from_path(
                         self, artifact, artifact_type=artifact_type, artifact_name=artifact_name
                     )
                 else:
+                    artifact = self._refang(artifact)
                     artifact = resources.LocalArtifact.from_content(
                         self, artifact, artifact_name=artifact_name or artifact,
                         artifact_type=artifact_type,
@@ -1443,10 +1524,18 @@ class PolySwarmAsyncAPI:
                     self, artifact, artifact_type=artifact_type, artifact_name=artifact_name
                 )
             elif artifact_type == resources.ArtifactType.URL:
-                artifact = resources.LocalArtifact.from_content(
-                    self, artifact, artifact_name=artifact_name or artifact,
-                    artifact_type=artifact_type,
-                )
+                # A QR-code submission's argument is an image path, not a URL
+                # (same rule as ``submit``): read the image, never refang it.
+                if preprocessing and preprocessing.get("type") == "qrcode":
+                    artifact = resources.LocalArtifact.from_path(
+                        self, artifact, artifact_type=artifact_type, artifact_name=artifact_name
+                    )
+                else:
+                    artifact = self._refang(artifact)
+                    artifact = resources.LocalArtifact.from_content(
+                        self, artifact, artifact_name=artifact_name or artifact,
+                        artifact_type=artifact_type,
+                    )
 
         json_params = {
             "artifact_name": artifact.artifact_name,
@@ -1505,6 +1594,7 @@ class PolySwarmAsyncAPI:
             else:
                 local = artifact
         else:
+            url = self._refang(url)
             local = resources.LocalArtifact.from_content(
                 self, url, artifact_name=artifact_name or url,
                 artifact_type=resources.ArtifactType.URL,

{polyswarm_api-4.4.0 → polyswarm_api-4.7.0}/src/polyswarm_api/api.py

@@ -19,7 +19,7 @@ import time
 import io
 import logging
 
-from polyswarm_api import exceptions, resources, settings
+from polyswarm_api import exceptions, refang, resources, settings
 from polyswarm_api.core import PolyswarmRequest, _as_result_bound
 
 from .session import PolyswarmSession
@@ -55,6 +55,7 @@ class PolyswarmAPI:
         verify: bool = True,
         *,
         session: PolyswarmSession | None = None,
+        refang_iocs: bool = False,
         **httpx_kwargs,
     ):
         key_masked = "******" + (key[-4:] if key and len(key) > 16 else "")
@@ -69,6 +70,11 @@ class PolyswarmAPI:
         self.community = community or settings.DEFAULT_COMMUNITY
         self.timeout = timeout or settings.DEFAULT_HTTP_TIMEOUT
         self.verify = verify
+        # Refang defanged URL / domain / IP inputs (``hxxps[:]//evil[.]com``)
+        # before building a request. Opt-in: the default preserves the 4.5.0
+        # behaviour of sending every input verbatim. See ``polyswarm_api.refang``
+        # and ``_refang`` below for exactly which inputs are touched.
+        self.refang_iocs = refang_iocs
         self._engines = None
         # Either accept a pre-built session (customization point) or
         # build the default from ``key``. Passing both is ambiguous.
@@ -95,6 +101,27 @@ class PolyswarmAPI:
         )
         return f"<{clsname}({attrs}) at 0x{id(self):x}>"
 
+    def _refang(self, value):
+        """Refang one URL / domain / IP input when ``refang_iocs`` is on.
+
+        Applied only to arguments that name a network indicator (a URL to
+        search or submit, an ``ips=`` / ``urls=`` / ``domains=`` entry, an IoC
+        ``ip`` / ``domain``, a known-host ``host``) — never to a free-form
+        metadata query or a hash.
+        Anything that is not a defanged network IoC is returned unchanged.
+        """
+        if not self.refang_iocs:
+            return value
+        return refang.refang_ioc(value)
+
+    def _refang_all(self, values):
+        """``_refang`` over a list argument; ``None`` / empty pass through."""
+        if not values or not self.refang_iocs:
+            return values
+        if isinstance(values, str):
+            return self._refang(values)
+        return [self._refang(v) for v in values]
+
     def close(self):
         """Close the underlying HTTP client."""
         self.session.close()
@@ -366,6 +393,7 @@ class PolyswarmAPI:
         :param url: A url to be searched by exact match
         :return: Generator of ArtifactInstance resources
         """
+        url = self._refang(url)
         logger.info("Searching for url %s", url)
         for item in self._paginate(resources.ArtifactInstance.search_url(self, url)):
             yield item
@@ -395,6 +423,11 @@ class PolyswarmAPI:
         :param exclude: A list of fields to be excluded from the result (.* wildcards are accepted)
         :return: Generator of ArtifactInstance resources
         """
+        ips, urls, domains = (
+            self._refang_all(ips),
+            self._refang_all(urls),
+            self._refang_all(domains),
+        )
         logger.info("Searching for metadata %s", query)
         for item in self._paginate(
             resources.Metadata.get(
@@ -426,7 +459,9 @@ class PolyswarmAPI:
         ):
             yield item
 
-    def search_by_ioc(self, ip=None, domain=None, ttp=None, imphash=None):
+    def search_by_ioc(
+        self, ip=None, domain=None, ttp=None, imphash=None, with_artifacts=False
+    ):
         """
         Search artifacts by IOC (ip, domain, ttp, or imphash)
 
@@ -434,14 +469,36 @@ class PolyswarmAPI:
         :param domain: domain address to search by
         :param ttp: ttp to search by
         :param imphash: ImpHash to search by
-        :return: Generator of ArtifactInstance resources
-        """
+        :param with_artifacts: True yields a Metadata resource per matching artifact
+            (a metadata-search row trimmed to a field set the server owns: artifact.*,
+            the scan summary, ssdeep/tlsh and the malware family) instead of its bare
+            sha256. With no ip/domain/ttp/imphash the server refuses it with a 400
+            (a typed exception); without it a bare call behaves as it always has.
+            Needs a server that supports the parameter:
+            an older one ignores it and answers bare sha256 strings, which fail to
+            parse as Metadata (TypeError).
+        :return: Generator of IOC resources whose ``json`` is a sha256 string, or of
+            Metadata resources when ``with_artifacts`` is True
+        """
+        ip, domain = self._refang(ip), self._refang(domain)
         logger.info(
-            "Searching by ioc %s", dict(ip=ip, domain=domain, ttp=ttp, imphash=imphash)
+            "Searching by ioc %s",
+            dict(
+                ip=ip,
+                domain=domain,
+                ttp=ttp,
+                imphash=imphash,
+                with_artifacts=with_artifacts,
+            ),
         )
         for item in self._paginate(
             resources.IOC.ioc_search(
-                self, ip=ip, domain=domain, ttp=ttp, imphash=imphash
+                self,
+                ip=ip,
+                domain=domain,
+                ttp=ttp,
+                imphash=imphash,
+                with_artifacts=with_artifacts,
             )
         ):
             yield item
@@ -454,6 +511,7 @@ class PolyswarmAPI:
         :param domains
         :return: Generator of IOC resources
         """
+        ips, domains = self._refang_all(ips), self._refang_all(domains)
         logger.info("Checking known hosts ips: %s, domains: %s", ips, domains)
         for item in self._paginate(resources.IOC.check_known_hosts(self, ips, domains)):
             yield item
@@ -467,6 +525,7 @@ class PolyswarmAPI:
         :param host
         :return: IOC resource
         """
+        host = self._refang(host)
         logger.info("Creating known good ioc %s %s %s", type, host, source)
         return self._single(resources.IOC.create_known_good(self, type, host, source))
 
@@ -479,6 +538,7 @@ class PolyswarmAPI:
         :param host
         :return: IOC resource
         """
+        host = self._refang(host)
         logger.info("Creating known bad ioc %s %s %s", type, host, source)
         return self._single(resources.IOC.create_known_bad(self, type, host, source))
 
@@ -491,6 +551,7 @@ class PolyswarmAPI:
         :param host
         :return: IOC resource
         """
+        host = self._refang(host)
         logger.info("Updating known good ioc %s %s %s %s", id, type, host, source)
         return self._single(
             resources.IOC.update_known_good(self, id, type, host, source, good)
@@ -837,7 +898,13 @@ class PolyswarmAPI:
         )
 
     def ruleset_list(
-        self, name=None, status=None, favorites_only=None, has_new_results=None
+        self,
+        name=None,
+        status=None,
+        favorites_only=None,
+        has_new_results=None,
+        sort=None,
+        exclude_favorites=None,
     ):
         """
         List all YaraRulesets for the current account.
@@ -847,11 +914,45 @@ class PolyswarmAPI:
         :param status: 'active' returns only rulesets whose live hunt is
             currently running.
         :param favorites_only: True returns only favorited rulesets.
+        :param exclude_favorites: True returns only the rulesets that are NOT
+            favorited — the inverse of ``favorites_only``, and refused together
+            with it (a contradiction, answered with an error rather than an
+            empty list). It exists for clients that render the favorites as
+            their own list: the favorites are a separate, unpaginated fetch
+            bounded by the account's budget, so leaving them in the paginated
+            list too makes a page either repeat a row or come back short.
+            Appended to the signature rather than placed beside
+            ``favorites_only`` so a positional caller keeps working.
         :param has_new_results: True returns only rulesets whose stored
             new-results counter is positive. The counter (and its window) is
             maintained server-side by a scheduled refresh; rows carry it as
             ``new_results_count`` with ``new_results_counted_at`` marking when
             it was last refreshed. There is no per-request window parameter.
+        :param sort: ``'active_first'`` returns the rulesets that carry a live
+            hunt link first, newest first within each block. Default (None) is
+            newest first. "Newest first" is the server's own insertion key, NOT
+            the ``id`` on the rows you get back — that one is unique but
+            unordered, so dedupe with it and never resume or bound a walk with
+            it. Applied SERVER-side, across pages — the list is
+            keyset-paginated, so a client-side sort would only ever reorder one
+            page; the SDK never re-orders rows. Reuse a page's ``offset`` only
+            with the same ``sort``: the server refuses a cursor minted under
+            the other order.
+
+            Two server-side properties of that key, neither of them SDK
+            behaviour. It ranks on the stored link, which is a WIDER predicate
+            than the one ``livescan_id`` is rendered under: a legacy row whose
+            hunt was stopped without clearing the link ranks in the leading
+            block while still serializing ``livescan_id`` as ``None``. Read the
+            field to decide whether a ruleset is running; never the position.
+
+            And the key is MUTABLE, unlike that default: a ruleset whose
+            live hunt stops mid-walk falls back into the idle block below the
+            cursor and is yielded twice, and one started mid-walk moves above
+            the cursor and is skipped for the rest of that walk. That is a
+            property of the walk, so starting fresh from the first page does
+            not avoid it. This generator streams pages and does not dedupe —
+            dedupe by ``id`` if you consume more than one page.
         :return: A generator of YaraRuleset resources
         """
         logger.info("List rulesets")
@@ -862,6 +963,8 @@ class PolyswarmAPI:
                 status=status,
                 favorites_only=favorites_only,
                 has_new_results=has_new_results,
+                sort=sort,
+                exclude_favorites=exclude_favorites,
                 community=self.community,
             )
         ):
@@ -1658,7 +1761,7 @@ class PolyswarmAPI:
                     artifact_name=artifact_name,
                 )
             elif artifact_type == resources.ArtifactType.URL:
-                if preprocessing and preprocessing["type"] == "qrcode":
+                if preprocessing and preprocessing.get("type") == "qrcode":
                     artifact = resources.LocalArtifact.from_path(
                         self,
                         artifact,
@@ -1666,6 +1769,7 @@ class PolyswarmAPI:
                         artifact_name=artifact_name,
                     )
                 else:
+                    artifact = self._refang(artifact)
                     artifact = resources.LocalArtifact.from_content(
                         self,
                         artifact,
@@ -1780,12 +1884,23 @@ class PolyswarmAPI:
                     artifact_name=artifact_name,
                 )
             elif artifact_type == resources.ArtifactType.URL:
-                artifact = resources.LocalArtifact.from_content(
-                    self,
-                    artifact,
-                    artifact_name=artifact_name or artifact,
-                    artifact_type=artifact_type,
-                )
+                # A QR-code submission's argument is an image path, not a URL
+                # (same rule as ``submit``): read the image, never refang it.
+                if preprocessing and preprocessing.get("type") == "qrcode":
+                    artifact = resources.LocalArtifact.from_path(
+                        self,
+                        artifact,
+                        artifact_type=artifact_type,
+                        artifact_name=artifact_name,
+                    )
+                else:
+                    artifact = self._refang(artifact)
+                    artifact = resources.LocalArtifact.from_content(
+                        self,
+                        artifact,
+                        artifact_name=artifact_name or artifact,
+                        artifact_type=artifact_type,
+                    )
 
         json_params = {
             "artifact_name": artifact.artifact_name,
@@ -1854,6 +1969,7 @@ class PolyswarmAPI:
             else:
                 local = artifact
         else:
+            url = self._refang(url)
             local = resources.LocalArtifact.from_content(
                 self,
                 url,

polyswarm_api-4.7.0/src/polyswarm_api/refang.py

@@ -0,0 +1,113 @@
+"""Refang defanged indicators of compromise (IoCs).
+
+Portability is part of the contract, because the same pattern can match
+different characters in different regex engines. So: no case-insensitive
+flag (Python's folds Unicode, e.g. U+212A KELVIN SIGN matches ``k``) — letters
+are spelled as explicit ``[aA]`` classes; no ``\\b``, ``\\d``, ``\\w``, ``\\s``
+or ``\\S`` (their Unicode sets differ between engines) — whitespace is the
+explicit ASCII set ``[ \\t\\n\\r\\f\\v]``, and trimming strips only that set;
+and full matches use ``fullmatch`` (Python's ``$`` also matches before a
+trailing newline).
+
+What the contract is, why clients refang at their own edge, where the client
+applies it and what is out of scope: ``specs/05-downstream-contract.md``
+§"IoC refanging".
+"""
+
+import re
+
+__all__ = ['is_network_ioc', 'refang_ioc', 'refang_text']
+
+# ASCII whitespace only — see the module docstring on portability.
+_WS_CHARS = ' \t\n\r\f\v'
+_WS = r'[ \t\n\r\f\v]'
+
+# Applied in order. The bracket rules run first so that ``hxxps[:]//`` has
+# become ``hxxps://`` by the time the (anchored) scheme rules look for ``://``.
+# ``hxxps`` is matched before ``hxxp``: the shorter rule would otherwise leave
+# a stray ``s`` behind. Each scheme rule is a literal replacement — a capture
+# group would carry the input's case (``HXXPS`` -> ``httpS``).
+_RULES = (
+    (re.compile(rf'[\[({{]{_WS}*:{_WS}*/{_WS}*/{_WS}*[\])}}]'), '://'),
+    (re.compile(rf'[\[({{]{_WS}*:{_WS}*[\])}}]'), ':'),
+    (re.compile(rf'[\[({{]{_WS}*/{_WS}*[\])}}]'), '/'),
+    (re.compile(rf'[\[({{]{_WS}*\.{_WS}*[\])}}]'), '.'),
+    (re.compile(rf'[\[({{]{_WS}*[dD][oO][tT]{_WS}*[\])}}]'), '.'),
+    (re.compile(r'^[hH][xX*]{2}[pP][sS](?=://)'), 'https'),
+    (re.compile(r'^[hH][xX*]{2}[pP](?=://)'), 'http'),
+    (re.compile(r'^[fF][xX][pP][sS](?=://)'), 'ftps'),
+    (re.compile(r'^[fF][xX][pP](?=://)'), 'ftp'),
+)
+
+_OCTET = r'(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])'
+_IPV4 = rf'{_OCTET}(?:\.{_OCTET}){{3}}'
+_LABEL = r'[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?'
+_TLD = r'(?:[a-zA-Z]{2,63}|[xX][nN]--[a-zA-Z0-9-]{1,59})'
+_DOMAIN = rf'(?:{_LABEL}\.)+{_TLD}'
+_IPV6 = r'\[[0-9a-fA-F:.]+\]'
+_HOST = rf'(?:{_DOMAIN}|{_IPV4}|{_IPV6})'
+_SCHEME = r'(?:[hH][tT][tT][pP][sS]?|[fF][tT][pP][sS]?)'
+# Used with ``fullmatch`` — never ``^…$``, whose ``$`` accepts a trailing ``\n``.
+_NETWORK_IOC = re.compile(
+    rf'(?:{_SCHEME}://)?(?:[^ \t\n\r\f\v/?#@]+@)?{_HOST}(?::[0-9]{{1,5}})?(?:[/?#][^ \t\n\r\f\v]*)?'
+)
+# An optional scheme-like token (live or defanged, e.g. ``hxxps``) plus the
+# host: everything before the first ``/``, ``?`` or ``#``.
+_SCHEME_AND_HOST = re.compile(r'^(?:[a-zA-Z*]+://)?[^/?#]*')
+_QUERY_SYNTAX = re.compile(r'[ \t\n\r\f\v"]')
+
+
+def refang_text(text):
+    """Apply every refang rewrite to ``text``, with no trimming and no gate.
+
+    Ungated: it rewrites anything that contains a defang token, IoC or not.
+    Callers that take user input want :func:`refang_ioc` instead.
+    """
+    for pattern, replacement in _RULES:
+        text = pattern.sub(replacement, text)
+    return text
+
+
+def is_network_ioc(candidate):
+    """Whether ``candidate`` is a URL, a domain, or an IP address.
+
+    A URL may carry an http(s)/ftp(s) scheme, userinfo, a port and a path; an
+    IPv6 host is accepted only in its bracketed URL form.
+    """
+    return bool(_NETWORK_IOC.fullmatch(candidate))
+
+
+def refang_ioc(value, accept=None):
+    """Return the refanged form of ``value`` if it is a defanged network IoC.
+
+    Anything else comes back exactly as given (untrimmed), so this is safe to
+    run on every IoC-shaped input. The gate, in order:
+
+    * nothing was defanged -> unchanged;
+    * the rewrite contains ASCII whitespace or a double quote -> unchanged: that is
+      a query or quoted data, never a single indicator;
+    * the rewrite keeps the input's scheme and host intact (with or without
+      a scheme) -> unchanged: only a path, query or fragment would change,
+      so a legitimate ``[.]`` in ``example.com/a[.]b`` survives;
+    * the rewrite is not a URL, domain or IP -> unchanged;
+    * ``accept`` (optional) rejects the rewrite -> unchanged. Callers use it
+      to require their own routing to agree, e.g. "this would be searched as
+      a URL".
+
+    A refanged value is returned trimmed of ASCII whitespace.
+    """
+    if not isinstance(value, str):
+        return value
+    trimmed = value.strip(_WS_CHARS)
+    candidate = refang_text(trimmed)
+    if candidate == trimmed:
+        return value
+    if _QUERY_SYNTAX.search(candidate):
+        return value
+    if candidate.startswith(_SCHEME_AND_HOST.match(trimmed).group(0)):
+        return value
+    if not is_network_ioc(candidate):
+        return value
+    if accept is not None and not accept(candidate):
+        return value
+    return candidate

{polyswarm_api-4.4.0 → polyswarm_api-4.7.0}/src/polyswarm_api/resources.py

@@ -169,7 +169,7 @@ class IOC(core.BaseJsonResource):
         )
 
     @classmethod
-    def ioc_search(cls, api, ip=None, domain=None, ttp=None, imphash=None):
+    def ioc_search(cls, api, ip=None, domain=None, ttp=None, imphash=None, with_artifacts=False):
         params = dict(community=api.community)
         if ip is not None:
             params['ip'] = ip
@@ -179,12 +179,17 @@ class IOC(core.BaseJsonResource):
             params['ttp'] = ttp
         if imphash is not None:
             params['imphash'] = imphash
+        if with_artifacts:
+            # An int, not a bool: the session renders bools as 'True', which the
+            # server's boolean parser refuses (it accepts only 0/1/false/true).
+            params['with_artifacts'] = 1
         return core.PolyswarmRequest(
             api=api,
             method='GET',
             url=f'{api.uri}/ioc/search',
             params=params,
-            result_parser=cls,
+            # Opt-in rows are metadata-search documents; the default is bare sha256s.
+            result_parser=Metadata if with_artifacts else cls,
         )
 
     @classmethod
@@ -805,6 +810,11 @@ class LiveHuntResult(core.BaseJsonResource):
         self.sha1 = content.get('sha1')
         self.rule_name = content['rule_name']
         self.tags = content['tags']
+        # `.get()`, not a subscript -- both keys are additive, so an older server omits
+        # them. None is AMBIGUOUS on both (four causes) and is never a claim that the
+        # evidence is complete. Contract: specs/05-downstream-contract.md.
+        self.matched_strings = content.get('matched_strings')
+        self.matched_strings_dropped = content.get('matched_strings_dropped')
         self.polyscore = content['polyscore']
         self.malware_family = content['malware_family']
         self.detections = content['detections']
@@ -863,6 +873,11 @@ class HistoricalHuntResult(core.BaseJsonResource):
         self.created = core.parse_isoformat(content['created'])
         self.rule_name = content['rule_name']
         self.tags = content['tags']
+        # `.get()`, not a subscript -- both keys are additive, so an older server omits
+        # them. None is AMBIGUOUS on both (four causes) and is never a claim that the
+        # evidence is complete. Contract: specs/05-downstream-contract.md.
+        self.matched_strings = content.get('matched_strings')
+        self.matched_strings_dropped = content.get('matched_strings_dropped')
         self.polyscore = content['polyscore']
         self.malware_family = content['malware_family']
         self.detections = content['detections']

{polyswarm_api-4.4.0 → polyswarm_api-4.7.0/src/polyswarm_api.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polyswarm_api
-Version: 4.4.0
+Version: 4.7.0
 Summary: Client library to simplify interacting with the PolySwarm consumer API
 Author-email: PolySwarm Developers <info@polyswarm.io>
 License: MIT

{polyswarm_api-4.4.0 → polyswarm_api-4.7.0}/src/polyswarm_api.egg-info/SOURCES.txt

@@ -5,6 +5,7 @@ src/polyswarm_api/__init__.py
 src/polyswarm_api/api.py
 src/polyswarm_api/core.py
 src/polyswarm_api/exceptions.py
+src/polyswarm_api/refang.py
 src/polyswarm_api/resources.py
 src/polyswarm_api/session.py
 src/polyswarm_api/settings.py