polyswarm-api 4.5.0__tar.gz → 4.8.0__tar.gz

{polyswarm_api-4.5.0/src/polyswarm_api.egg-info → polyswarm_api-4.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polyswarm_api
-Version: 4.5.0
+Version: 4.8.0
 Summary: Client library to simplify interacting with the PolySwarm consumer API
 Author-email: PolySwarm Developers <info@polyswarm.io>
 License: MIT

{polyswarm_api-4.5.0 → polyswarm_api-4.8.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "polyswarm_api"
-version = "4.5.0"
+version = "4.8.0"
 description = "Client library to simplify interacting with the PolySwarm consumer API"
 readme = "README.md"
 requires-python = ">=3.10,<4"
@@ -55,7 +55,7 @@ package-dir = { "" = "src" }
 where = ["src"]
 
 [tool.bumpversion]
-current_version = "4.5.0"
+current_version = "4.8.0"
 commit = true
 tag = false
 sign_tags = true

{polyswarm_api-4.5.0 → polyswarm_api-4.8.0}/src/polyswarm_api/__init__.py

@@ -1,9 +1,10 @@
 # https://www.python.org/dev/peps/pep-0008/#module-level-dunder-names
-__version__ = '4.5.0'
+__version__ = '4.8.0'
 __release_url__ = 'https://api.github.com/repos/polyswarm/polyswarm-api/releases/latest'
 
 from . import api
 from . import exceptions
+from . import refang
 from .api import PolyswarmAPI
 from .session import PolyswarmSession
 from .aio import PolySwarmAsyncAPI, AsyncPolyswarmSession

{polyswarm_api-4.5.0 → polyswarm_api-4.8.0}/src/polyswarm_api/aio/api.py

@@ -17,7 +17,7 @@ import io
 import logging
 import time
 
-from polyswarm_api import exceptions, resources, settings
+from polyswarm_api import exceptions, refang, resources, settings
 from polyswarm_api.core import PolyswarmRequest, _as_result_bound
 
 from .session import AsyncPolyswarmSession
@@ -53,6 +53,7 @@ class PolySwarmAsyncAPI:
         verify: bool = True,
         *,
         session: AsyncPolyswarmSession | None = None,
+        refang_iocs: bool = False,
         **httpx_kwargs,
     ):
         key_masked = '******' + (key[-4:] if key and len(key) > 16 else '')
@@ -64,6 +65,11 @@ class PolySwarmAsyncAPI:
         self.community = community or settings.DEFAULT_COMMUNITY
         self.timeout = timeout or settings.DEFAULT_HTTP_TIMEOUT
         self.verify = verify
+        # Refang defanged URL / domain / IP inputs (``hxxps[:]//evil[.]com``)
+        # before building a request. Opt-in: the default preserves the 4.5.0
+        # behaviour of sending every input verbatim. See ``polyswarm_api.refang``
+        # and ``_refang`` below for exactly which inputs are touched.
+        self.refang_iocs = refang_iocs
         self._engines = None
         # Either accept a pre-built session (customization point) or
         # build the default from ``key``. Passing both is ambiguous.
@@ -88,6 +94,27 @@ class PolySwarmAsyncAPI:
         attrs = f'uri={self.uri!r}, community={self.community!r}, timeout={self.timeout!r}'
         return f'<{clsname}({attrs}) at 0x{id(self):x}>'
 
+    def _refang(self, value):
+        """Refang one URL / domain / IP input when ``refang_iocs`` is on.
+
+        Applied only to arguments that name a network indicator (a URL to
+        search or submit, an ``ips=`` / ``urls=`` / ``domains=`` entry, an IoC
+        ``ip`` / ``domain``, a known-host ``host``) — never to a free-form
+        metadata query or a hash.
+        Anything that is not a defanged network IoC is returned unchanged.
+        """
+        if not self.refang_iocs:
+            return value
+        return refang.refang_ioc(value)
+
+    def _refang_all(self, values):
+        """``_refang`` over a list argument; ``None`` / empty pass through."""
+        if not values or not self.refang_iocs:
+            return values
+        if isinstance(values, str):
+            return self._refang(values)
+        return [self._refang(v) for v in values]
+
     async def aclose(self):
         """Close the underlying HTTP client."""
         await self.session.aclose()
@@ -261,6 +288,51 @@ class PolySwarmAsyncAPI:
             result_parser=resources.MetadataMapping,
         )
 
+    async def metadata_raw_search(self, body, community=None):
+        """Run a raw OpenSearch query body against the metadata corpus.
+
+        The server overlays the community filter itself and caps the work, so
+        ``body`` is a plain ``_search`` body and the community travels in the
+        query string rather than in it. The accepted grammar is closed: a
+        construct the server does not recognise is refused rather than passed
+        through, and so is a top-level ``community``/``communities`` key.
+
+        Entitlement-gated and metered per query, including a query the server
+        refuses after running it.
+
+        :param body: An OpenSearch ``_search`` body (``query``, ``aggs``,
+            ``size``, ``sort``, ``_source``, ...). Omitting ``size`` returns
+            **no documents**: the server defaults it to 0 so an aggregation
+            caller does not pay for hits they did not ask for. Ask for hits
+            by setting it.
+        :param community: Community to search; defaults to the client's.
+        :return: A ``MetadataRawSearch`` whose ``json`` carries the engine's
+            ``hits`` and ``aggregations``, plus ``partial`` when the server
+            could not establish that the answer is complete.
+
+            ``partial`` is a **list to inspect, not a flag**. Its entries are
+            ``{'code': ..., 'message': ...}`` and they make different claims,
+            so branch on ``code`` - ``message`` is for a human and may be
+            reworded. The codes are a closed set:
+
+            * ``timed_out`` / ``terminated_early`` - the search stopped early.
+            * ``shards_failed`` - a shard was lost; also carries ``failed``.
+            * ``buckets_clamped`` - a ``terms`` list dropped terms.
+            * ``composite_paged`` - a ``composite`` may have more pages; page
+              it with its ``after_key``.
+            * ``bucket_count_approximate`` - a ``terms`` count is approximate
+              rather than truncated.
+            * ``hit_total_capped`` - the hit counter stopped at
+              ``track_total_hits``. This is what *every* large query that asks
+              for hits returns, so on its own it says nothing about the
+              aggregations; the server omits it when the body asked for no
+              documents. ``hits.total.relation`` carries the same fact as the
+              engine's own field, ``eq`` exact and ``gte`` capped.
+        """
+        logger.info('Running a raw metadata search')
+        return await self._single(
+            resources.MetadataRawSearch.search(self, body, community))
+
     async def metadata_field_properties_write(self, field_path, description,
                                         example=None, category=None, aliases=None):
         """Upsert a metadata field properties entry (the single write path).
@@ -340,6 +412,7 @@ class PolySwarmAsyncAPI:
         :param url: A url to be searched by exact match
         :return: Generator of ArtifactInstance resources
         """
+        url = self._refang(url)
         logger.info('Searching for url %s', url)
         async for item in self._paginate(resources.ArtifactInstance.search_url(self, url)):
             yield item
@@ -365,6 +438,7 @@ class PolySwarmAsyncAPI:
         :param exclude: A list of fields to be excluded from the result (.* wildcards are accepted)
         :return: Generator of ArtifactInstance resources
         """
+        ips, urls, domains = self._refang_all(ips), self._refang_all(urls), self._refang_all(domains)
         logger.info('Searching for metadata %s', query)
         async for item in self._paginate(resources.Metadata.get(self, query=query, community=self.community, include=include, exclude=exclude, ips=ips, urls=urls, domains=domains)):
             yield item
@@ -381,18 +455,28 @@ class PolySwarmAsyncAPI:
         async for item in self._paginate(resources.IOC.iocs_by_hash(self, hash_value, hash_type, hide_known_good=hide_known_good, beta=beta)):
             yield item
 
-    async def search_by_ioc(self, ip=None, domain=None, ttp=None, imphash=None):
+    async def search_by_ioc(self, ip=None, domain=None, ttp=None, imphash=None, with_artifacts=False):
         """
         Search artifacts by IOC (ip, domain, ttp, or imphash)
-        
+
         :param ip: ip address to search by
         :param domain: domain address to search by
         :param ttp: ttp to search by
         :param imphash: ImpHash to search by
-        :return: Generator of ArtifactInstance resources
-        """
-        logger.info('Searching by ioc %s', dict(ip=ip, domain=domain, ttp=ttp, imphash=imphash))
-        async for item in self._paginate(resources.IOC.ioc_search(self, ip=ip, domain=domain, ttp=ttp, imphash=imphash)):
+        :param with_artifacts: True yields a Metadata resource per matching artifact
+            (a metadata-search row trimmed to a field set the server owns: artifact.*,
+            the scan summary, ssdeep/tlsh and the malware family) instead of its bare
+            sha256. With no ip/domain/ttp/imphash the server refuses it with a 400
+            (a typed exception); without it a bare call behaves as it always has.
+            Needs a server that supports the parameter:
+            an older one ignores it and answers bare sha256 strings, which fail to
+            parse as Metadata (TypeError).
+        :return: Generator of IOC resources whose ``json`` is a sha256 string, or of
+            Metadata resources when ``with_artifacts`` is True
+        """
+        ip, domain = self._refang(ip), self._refang(domain)
+        logger.info('Searching by ioc %s', dict(ip=ip, domain=domain, ttp=ttp, imphash=imphash, with_artifacts=with_artifacts))
+        async for item in self._paginate(resources.IOC.ioc_search(self, ip=ip, domain=domain, ttp=ttp, imphash=imphash, with_artifacts=with_artifacts)):
             yield item
 
     async def check_known_hosts(self, ips=[], domains=[]):
@@ -403,6 +487,7 @@ class PolySwarmAsyncAPI:
         :param domains
         :return: Generator of IOC resources
         """
+        ips, domains = self._refang_all(ips), self._refang_all(domains)
         logger.info('Checking known hosts ips: %s, domains: %s', ips, domains)
         async for item in self._paginate(resources.IOC.check_known_hosts(self, ips, domains)):
             yield item
@@ -416,6 +501,7 @@ class PolySwarmAsyncAPI:
         :param host
         :return: IOC resource
         """
+        host = self._refang(host)
         logger.info('Creating known good ioc %s %s %s', type, host, source)
         return await self._single(resources.IOC.create_known_good(self, type, host, source))
 
@@ -428,6 +514,7 @@ class PolySwarmAsyncAPI:
         :param host
         :return: IOC resource
         """
+        host = self._refang(host)
         logger.info('Creating known bad ioc %s %s %s', type, host, source)
         return await self._single(resources.IOC.create_known_bad(self, type, host, source))
 
@@ -440,6 +527,7 @@ class PolySwarmAsyncAPI:
         :param host
         :return: IOC resource
         """
+        host = self._refang(host)
         logger.info('Updating known good ioc %s %s %s %s', id, type, host, source)
         return await self._single(resources.IOC.update_known_good(self, id, type, host, source, good))
 
@@ -554,18 +642,21 @@ class PolySwarmAsyncAPI:
         """
         return await self._single(resources.LiveHuntResult.get(self, id=result_id))
 
-    async def historical_create(self, rule=None, ruleset_name=None):
+    async def historical_create(self, rule=None, ruleset_name=None, window_days=None):
         """
         Run a new historical hunt.
 
         :param rule: YaraRuleset object or string containing YARA rules to install
         :param ruleset_name: Name of the ruleset.
+        :param window_days: Days back the hunt reaches: 30, 60, 90 or 180. Omit it for the
+            platform maximum. The server validates the value; the SDK does not.
         :return: The created Hunt resource
         """
         logger.info('Create historical hunt %s', rule)
         rule, rule_id = self._parse_rule(rule)
         return await self._single(resources.HistoricalHunt.create(self, yara=rule.yara if rule else None, rule_id=rule_id,
-                                               ruleset_name=ruleset_name, community=self.community))
+                                               ruleset_name=ruleset_name, window_days=window_days,
+                                               community=self.community))
 
     async def historical_get(self, hunt=None):
         """
@@ -1373,11 +1464,12 @@ class PolySwarmAsyncAPI:
                     self, artifact, artifact_type=artifact_type, artifact_name=artifact_name
                 )
             elif artifact_type == resources.ArtifactType.URL:
-                if preprocessing and preprocessing["type"] == "qrcode":
+                if preprocessing and preprocessing.get("type") == "qrcode":
                     artifact = resources.LocalArtifact.from_path(
                         self, artifact, artifact_type=artifact_type, artifact_name=artifact_name
                     )
                 else:
+                    artifact = self._refang(artifact)
                     artifact = resources.LocalArtifact.from_content(
                         self, artifact, artifact_name=artifact_name or artifact,
                         artifact_type=artifact_type,
@@ -1480,10 +1572,18 @@ class PolySwarmAsyncAPI:
                     self, artifact, artifact_type=artifact_type, artifact_name=artifact_name
                 )
             elif artifact_type == resources.ArtifactType.URL:
-                artifact = resources.LocalArtifact.from_content(
-                    self, artifact, artifact_name=artifact_name or artifact,
-                    artifact_type=artifact_type,
-                )
+                # A QR-code submission's argument is an image path, not a URL
+                # (same rule as ``submit``): read the image, never refang it.
+                if preprocessing and preprocessing.get("type") == "qrcode":
+                    artifact = resources.LocalArtifact.from_path(
+                        self, artifact, artifact_type=artifact_type, artifact_name=artifact_name
+                    )
+                else:
+                    artifact = self._refang(artifact)
+                    artifact = resources.LocalArtifact.from_content(
+                        self, artifact, artifact_name=artifact_name or artifact,
+                        artifact_type=artifact_type,
+                    )
 
         json_params = {
             "artifact_name": artifact.artifact_name,
@@ -1542,6 +1642,7 @@ class PolySwarmAsyncAPI:
             else:
                 local = artifact
         else:
+            url = self._refang(url)
             local = resources.LocalArtifact.from_content(
                 self, url, artifact_name=artifact_name or url,
                 artifact_type=resources.ArtifactType.URL,

{polyswarm_api-4.5.0 → polyswarm_api-4.8.0}/src/polyswarm_api/api.py

@@ -19,7 +19,7 @@ import time
 import io
 import logging
 
-from polyswarm_api import exceptions, resources, settings
+from polyswarm_api import exceptions, refang, resources, settings
 from polyswarm_api.core import PolyswarmRequest, _as_result_bound
 
 from .session import PolyswarmSession
@@ -55,6 +55,7 @@ class PolyswarmAPI:
         verify: bool = True,
         *,
         session: PolyswarmSession | None = None,
+        refang_iocs: bool = False,
         **httpx_kwargs,
     ):
         key_masked = "******" + (key[-4:] if key and len(key) > 16 else "")
@@ -69,6 +70,11 @@ class PolyswarmAPI:
         self.community = community or settings.DEFAULT_COMMUNITY
         self.timeout = timeout or settings.DEFAULT_HTTP_TIMEOUT
         self.verify = verify
+        # Refang defanged URL / domain / IP inputs (``hxxps[:]//evil[.]com``)
+        # before building a request. Opt-in: the default preserves the 4.5.0
+        # behaviour of sending every input verbatim. See ``polyswarm_api.refang``
+        # and ``_refang`` below for exactly which inputs are touched.
+        self.refang_iocs = refang_iocs
         self._engines = None
         # Either accept a pre-built session (customization point) or
         # build the default from ``key``. Passing both is ambiguous.
@@ -95,6 +101,27 @@ class PolyswarmAPI:
         )
         return f"<{clsname}({attrs}) at 0x{id(self):x}>"
 
+    def _refang(self, value):
+        """Refang one URL / domain / IP input when ``refang_iocs`` is on.
+
+        Applied only to arguments that name a network indicator (a URL to
+        search or submit, an ``ips=`` / ``urls=`` / ``domains=`` entry, an IoC
+        ``ip`` / ``domain``, a known-host ``host``) — never to a free-form
+        metadata query or a hash.
+        Anything that is not a defanged network IoC is returned unchanged.
+        """
+        if not self.refang_iocs:
+            return value
+        return refang.refang_ioc(value)
+
+    def _refang_all(self, values):
+        """``_refang`` over a list argument; ``None`` / empty pass through."""
+        if not values or not self.refang_iocs:
+            return values
+        if isinstance(values, str):
+            return self._refang(values)
+        return [self._refang(v) for v in values]
+
     def close(self):
         """Close the underlying HTTP client."""
         self.session.close()
@@ -276,6 +303,50 @@ class PolyswarmAPI:
             result_parser=resources.MetadataMapping,
         )
 
+    def metadata_raw_search(self, body, community=None):
+        """Run a raw OpenSearch query body against the metadata corpus.
+
+        The server overlays the community filter itself and caps the work, so
+        ``body`` is a plain ``_search`` body and the community travels in the
+        query string rather than in it. The accepted grammar is closed: a
+        construct the server does not recognise is refused rather than passed
+        through, and so is a top-level ``community``/``communities`` key.
+
+        Entitlement-gated and metered per query, including a query the server
+        refuses after running it.
+
+        :param body: An OpenSearch ``_search`` body (``query``, ``aggs``,
+            ``size``, ``sort``, ``_source``, ...). Omitting ``size`` returns
+            **no documents**: the server defaults it to 0 so an aggregation
+            caller does not pay for hits they did not ask for. Ask for hits
+            by setting it.
+        :param community: Community to search; defaults to the client's.
+        :return: A ``MetadataRawSearch`` whose ``json`` carries the engine's
+            ``hits`` and ``aggregations``, plus ``partial`` when the server
+            could not establish that the answer is complete.
+
+            ``partial`` is a **list to inspect, not a flag**. Its entries are
+            ``{'code': ..., 'message': ...}`` and they make different claims,
+            so branch on ``code`` - ``message`` is for a human and may be
+            reworded. The codes are a closed set:
+
+            * ``timed_out`` / ``terminated_early`` - the search stopped early.
+            * ``shards_failed`` - a shard was lost; also carries ``failed``.
+            * ``buckets_clamped`` - a ``terms`` list dropped terms.
+            * ``composite_paged`` - a ``composite`` may have more pages; page
+              it with its ``after_key``.
+            * ``bucket_count_approximate`` - a ``terms`` count is approximate
+              rather than truncated.
+            * ``hit_total_capped`` - the hit counter stopped at
+              ``track_total_hits``. This is what *every* large query that asks
+              for hits returns, so on its own it says nothing about the
+              aggregations; the server omits it when the body asked for no
+              documents. ``hits.total.relation`` carries the same fact as the
+              engine's own field, ``eq`` exact and ``gte`` capped.
+        """
+        logger.info("Running a raw metadata search")
+        return self._single(resources.MetadataRawSearch.search(self, body, community))
+
     def metadata_field_properties_write(
         self, field_path, description, example=None, category=None, aliases=None
     ):
@@ -366,6 +437,7 @@ class PolyswarmAPI:
         :param url: A url to be searched by exact match
         :return: Generator of ArtifactInstance resources
         """
+        url = self._refang(url)
         logger.info("Searching for url %s", url)
         for item in self._paginate(resources.ArtifactInstance.search_url(self, url)):
             yield item
@@ -395,6 +467,11 @@ class PolyswarmAPI:
         :param exclude: A list of fields to be excluded from the result (.* wildcards are accepted)
         :return: Generator of ArtifactInstance resources
         """
+        ips, urls, domains = (
+            self._refang_all(ips),
+            self._refang_all(urls),
+            self._refang_all(domains),
+        )
         logger.info("Searching for metadata %s", query)
         for item in self._paginate(
             resources.Metadata.get(
@@ -426,7 +503,9 @@ class PolyswarmAPI:
         ):
             yield item
 
-    def search_by_ioc(self, ip=None, domain=None, ttp=None, imphash=None):
+    def search_by_ioc(
+        self, ip=None, domain=None, ttp=None, imphash=None, with_artifacts=False
+    ):
         """
         Search artifacts by IOC (ip, domain, ttp, or imphash)
 
@@ -434,14 +513,36 @@ class PolyswarmAPI:
         :param domain: domain address to search by
         :param ttp: ttp to search by
         :param imphash: ImpHash to search by
-        :return: Generator of ArtifactInstance resources
-        """
+        :param with_artifacts: True yields a Metadata resource per matching artifact
+            (a metadata-search row trimmed to a field set the server owns: artifact.*,
+            the scan summary, ssdeep/tlsh and the malware family) instead of its bare
+            sha256. With no ip/domain/ttp/imphash the server refuses it with a 400
+            (a typed exception); without it a bare call behaves as it always has.
+            Needs a server that supports the parameter:
+            an older one ignores it and answers bare sha256 strings, which fail to
+            parse as Metadata (TypeError).
+        :return: Generator of IOC resources whose ``json`` is a sha256 string, or of
+            Metadata resources when ``with_artifacts`` is True
+        """
+        ip, domain = self._refang(ip), self._refang(domain)
         logger.info(
-            "Searching by ioc %s", dict(ip=ip, domain=domain, ttp=ttp, imphash=imphash)
+            "Searching by ioc %s",
+            dict(
+                ip=ip,
+                domain=domain,
+                ttp=ttp,
+                imphash=imphash,
+                with_artifacts=with_artifacts,
+            ),
         )
         for item in self._paginate(
             resources.IOC.ioc_search(
-                self, ip=ip, domain=domain, ttp=ttp, imphash=imphash
+                self,
+                ip=ip,
+                domain=domain,
+                ttp=ttp,
+                imphash=imphash,
+                with_artifacts=with_artifacts,
             )
         ):
             yield item
@@ -454,6 +555,7 @@ class PolyswarmAPI:
         :param domains
         :return: Generator of IOC resources
         """
+        ips, domains = self._refang_all(ips), self._refang_all(domains)
         logger.info("Checking known hosts ips: %s, domains: %s", ips, domains)
         for item in self._paginate(resources.IOC.check_known_hosts(self, ips, domains)):
             yield item
@@ -467,6 +569,7 @@ class PolyswarmAPI:
         :param host
         :return: IOC resource
         """
+        host = self._refang(host)
         logger.info("Creating known good ioc %s %s %s", type, host, source)
         return self._single(resources.IOC.create_known_good(self, type, host, source))
 
@@ -479,6 +582,7 @@ class PolyswarmAPI:
         :param host
         :return: IOC resource
         """
+        host = self._refang(host)
         logger.info("Creating known bad ioc %s %s %s", type, host, source)
         return self._single(resources.IOC.create_known_bad(self, type, host, source))
 
@@ -491,6 +595,7 @@ class PolyswarmAPI:
         :param host
         :return: IOC resource
         """
+        host = self._refang(host)
         logger.info("Updating known good ioc %s %s %s %s", id, type, host, source)
         return self._single(
             resources.IOC.update_known_good(self, id, type, host, source, good)
@@ -630,12 +735,14 @@ class PolyswarmAPI:
         """
         return self._single(resources.LiveHuntResult.get(self, id=result_id))
 
-    def historical_create(self, rule=None, ruleset_name=None):
+    def historical_create(self, rule=None, ruleset_name=None, window_days=None):
         """
         Run a new historical hunt.
 
         :param rule: YaraRuleset object or string containing YARA rules to install
         :param ruleset_name: Name of the ruleset.
+        :param window_days: Days back the hunt reaches: 30, 60, 90 or 180. Omit it for the
+            platform maximum. The server validates the value; the SDK does not.
         :return: The created Hunt resource
         """
         logger.info("Create historical hunt %s", rule)
@@ -646,6 +753,7 @@ class PolyswarmAPI:
                 yara=rule.yara if rule else None,
                 rule_id=rule_id,
                 ruleset_name=ruleset_name,
+                window_days=window_days,
                 community=self.community,
             )
         )
@@ -1700,7 +1808,7 @@ class PolyswarmAPI:
                     artifact_name=artifact_name,
                 )
             elif artifact_type == resources.ArtifactType.URL:
-                if preprocessing and preprocessing["type"] == "qrcode":
+                if preprocessing and preprocessing.get("type") == "qrcode":
                     artifact = resources.LocalArtifact.from_path(
                         self,
                         artifact,
@@ -1708,6 +1816,7 @@ class PolyswarmAPI:
                         artifact_name=artifact_name,
                     )
                 else:
+                    artifact = self._refang(artifact)
                     artifact = resources.LocalArtifact.from_content(
                         self,
                         artifact,
@@ -1822,12 +1931,23 @@ class PolyswarmAPI:
                     artifact_name=artifact_name,
                 )
             elif artifact_type == resources.ArtifactType.URL:
-                artifact = resources.LocalArtifact.from_content(
-                    self,
-                    artifact,
-                    artifact_name=artifact_name or artifact,
-                    artifact_type=artifact_type,
-                )
+                # A QR-code submission's argument is an image path, not a URL
+                # (same rule as ``submit``): read the image, never refang it.
+                if preprocessing and preprocessing.get("type") == "qrcode":
+                    artifact = resources.LocalArtifact.from_path(
+                        self,
+                        artifact,
+                        artifact_type=artifact_type,
+                        artifact_name=artifact_name,
+                    )
+                else:
+                    artifact = self._refang(artifact)
+                    artifact = resources.LocalArtifact.from_content(
+                        self,
+                        artifact,
+                        artifact_name=artifact_name or artifact,
+                        artifact_type=artifact_type,
+                    )
 
         json_params = {
             "artifact_name": artifact.artifact_name,
@@ -1896,6 +2016,7 @@ class PolyswarmAPI:
             else:
                 local = artifact
         else:
+            url = self._refang(url)
             local = resources.LocalArtifact.from_content(
                 self,
                 url,

polyswarm_api-4.8.0/src/polyswarm_api/refang.py

@@ -0,0 +1,113 @@
+"""Refang defanged indicators of compromise (IoCs).
+
+Portability is part of the contract, because the same pattern can match
+different characters in different regex engines. So: no case-insensitive
+flag (Python's folds Unicode, e.g. U+212A KELVIN SIGN matches ``k``) — letters
+are spelled as explicit ``[aA]`` classes; no ``\\b``, ``\\d``, ``\\w``, ``\\s``
+or ``\\S`` (their Unicode sets differ between engines) — whitespace is the
+explicit ASCII set ``[ \\t\\n\\r\\f\\v]``, and trimming strips only that set;
+and full matches use ``fullmatch`` (Python's ``$`` also matches before a
+trailing newline).
+
+What the contract is, why clients refang at their own edge, where the client
+applies it and what is out of scope: ``specs/05-downstream-contract.md``
+§"IoC refanging".
+"""
+
+import re
+
+__all__ = ['is_network_ioc', 'refang_ioc', 'refang_text']
+
+# ASCII whitespace only — see the module docstring on portability.
+_WS_CHARS = ' \t\n\r\f\v'
+_WS = r'[ \t\n\r\f\v]'
+
+# Applied in order. The bracket rules run first so that ``hxxps[:]//`` has
+# become ``hxxps://`` by the time the (anchored) scheme rules look for ``://``.
+# ``hxxps`` is matched before ``hxxp``: the shorter rule would otherwise leave
+# a stray ``s`` behind. Each scheme rule is a literal replacement — a capture
+# group would carry the input's case (``HXXPS`` -> ``httpS``).
+_RULES = (
+    (re.compile(rf'[\[({{]{_WS}*:{_WS}*/{_WS}*/{_WS}*[\])}}]'), '://'),
+    (re.compile(rf'[\[({{]{_WS}*:{_WS}*[\])}}]'), ':'),
+    (re.compile(rf'[\[({{]{_WS}*/{_WS}*[\])}}]'), '/'),
+    (re.compile(rf'[\[({{]{_WS}*\.{_WS}*[\])}}]'), '.'),
+    (re.compile(rf'[\[({{]{_WS}*[dD][oO][tT]{_WS}*[\])}}]'), '.'),
+    (re.compile(r'^[hH][xX*]{2}[pP][sS](?=://)'), 'https'),
+    (re.compile(r'^[hH][xX*]{2}[pP](?=://)'), 'http'),
+    (re.compile(r'^[fF][xX][pP][sS](?=://)'), 'ftps'),
+    (re.compile(r'^[fF][xX][pP](?=://)'), 'ftp'),
+)
+
+_OCTET = r'(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])'
+_IPV4 = rf'{_OCTET}(?:\.{_OCTET}){{3}}'
+_LABEL = r'[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?'
+_TLD = r'(?:[a-zA-Z]{2,63}|[xX][nN]--[a-zA-Z0-9-]{1,59})'
+_DOMAIN = rf'(?:{_LABEL}\.)+{_TLD}'
+_IPV6 = r'\[[0-9a-fA-F:.]+\]'
+_HOST = rf'(?:{_DOMAIN}|{_IPV4}|{_IPV6})'
+_SCHEME = r'(?:[hH][tT][tT][pP][sS]?|[fF][tT][pP][sS]?)'
+# Used with ``fullmatch`` — never ``^…$``, whose ``$`` accepts a trailing ``\n``.
+_NETWORK_IOC = re.compile(
+    rf'(?:{_SCHEME}://)?(?:[^ \t\n\r\f\v/?#@]+@)?{_HOST}(?::[0-9]{{1,5}})?(?:[/?#][^ \t\n\r\f\v]*)?'
+)
+# An optional scheme-like token (live or defanged, e.g. ``hxxps``) plus the
+# host: everything before the first ``/``, ``?`` or ``#``.
+_SCHEME_AND_HOST = re.compile(r'^(?:[a-zA-Z*]+://)?[^/?#]*')
+_QUERY_SYNTAX = re.compile(r'[ \t\n\r\f\v"]')
+
+
+def refang_text(text):
+    """Apply every refang rewrite to ``text``, with no trimming and no gate.
+
+    Ungated: it rewrites anything that contains a defang token, IoC or not.
+    Callers that take user input want :func:`refang_ioc` instead.
+    """
+    for pattern, replacement in _RULES:
+        text = pattern.sub(replacement, text)
+    return text
+
+
+def is_network_ioc(candidate):
+    """Whether ``candidate`` is a URL, a domain, or an IP address.
+
+    A URL may carry an http(s)/ftp(s) scheme, userinfo, a port and a path; an
+    IPv6 host is accepted only in its bracketed URL form.
+    """
+    return bool(_NETWORK_IOC.fullmatch(candidate))
+
+
+def refang_ioc(value, accept=None):
+    """Return the refanged form of ``value`` if it is a defanged network IoC.
+
+    Anything else comes back exactly as given (untrimmed), so this is safe to
+    run on every IoC-shaped input. The gate, in order:
+
+    * nothing was defanged -> unchanged;
+    * the rewrite contains ASCII whitespace or a double quote -> unchanged: that is
+      a query or quoted data, never a single indicator;
+    * the rewrite keeps the input's scheme and host intact (with or without
+      a scheme) -> unchanged: only a path, query or fragment would change,
+      so a legitimate ``[.]`` in ``example.com/a[.]b`` survives;
+    * the rewrite is not a URL, domain or IP -> unchanged;
+    * ``accept`` (optional) rejects the rewrite -> unchanged. Callers use it
+      to require their own routing to agree, e.g. "this would be searched as
+      a URL".
+
+    A refanged value is returned trimmed of ASCII whitespace.
+    """
+    if not isinstance(value, str):
+        return value
+    trimmed = value.strip(_WS_CHARS)
+    candidate = refang_text(trimmed)
+    if candidate == trimmed:
+        return value
+    if _QUERY_SYNTAX.search(candidate):
+        return value
+    if candidate.startswith(_SCHEME_AND_HOST.match(trimmed).group(0)):
+        return value
+    if not is_network_ioc(candidate):
+        return value
+    if accept is not None and not accept(candidate):
+        return value
+    return candidate

{polyswarm_api-4.5.0 → polyswarm_api-4.8.0}/src/polyswarm_api/resources.py

@@ -77,6 +77,25 @@ class MetadataMapping(core.BaseJsonResource):
     RESOURCE_ENDPOINT = '/search/metadata/mappings'
 
 
+class MetadataRawSearch(core.BaseJsonResource):
+    RESOURCE_ENDPOINT = '/search/metadata/raw'
+
+    @classmethod
+    def search(cls, api, body, community=None):
+        """``body`` is the caller's whole query and travels untouched. The
+        community is scope, not query, and the server reads it from the query
+        string - folding it into the body would pick scope with a key the
+        server's grammar then rejects."""
+        return core.PolyswarmRequest(
+            api=api,
+            method='POST',
+            url=f'{api.uri}{cls.RESOURCE_ENDPOINT}',
+            params={'community': community or api.community},
+            json=body,
+            result_parser=cls,
+        )
+
+
 class MetadataFieldProperties(core.BaseJsonResource):
     RESOURCE_ENDPOINT = '/search/metadata/properties'
     RESOURCE_ID_KEYS = ['field_path']
@@ -169,7 +188,7 @@ class IOC(core.BaseJsonResource):
         )
 
     @classmethod
-    def ioc_search(cls, api, ip=None, domain=None, ttp=None, imphash=None):
+    def ioc_search(cls, api, ip=None, domain=None, ttp=None, imphash=None, with_artifacts=False):
         params = dict(community=api.community)
         if ip is not None:
             params['ip'] = ip
@@ -179,12 +198,17 @@ class IOC(core.BaseJsonResource):
             params['ttp'] = ttp
         if imphash is not None:
             params['imphash'] = imphash
+        if with_artifacts:
+            # An int, not a bool: the session renders bools as 'True', which the
+            # server's boolean parser refuses (it accepts only 0/1/false/true).
+            params['with_artifacts'] = 1
         return core.PolyswarmRequest(
             api=api,
             method='GET',
             url=f'{api.uri}/ioc/search',
             params=params,
-            result_parser=cls,
+            # Opt-in rows are metadata-search documents; the default is bare sha256s.
+            result_parser=Metadata if with_artifacts else cls,
         )
 
     @classmethod
@@ -848,6 +872,8 @@ class HistoricalHunt(core.BaseJsonResource):
         # unknown (no source rule, or nothing to compare against) — not
         # "unchanged".
         self.source_rule_changed = content.get('source_rule_changed')
+        # None: no window requested (platform maximum), or a server predating the field.
+        self.window_days = content.get('window_days')
 
 
 class HistoricalHuntList(HistoricalHunt):

{polyswarm_api-4.5.0 → polyswarm_api-4.8.0/src/polyswarm_api.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polyswarm_api
-Version: 4.5.0
+Version: 4.8.0
 Summary: Client library to simplify interacting with the PolySwarm consumer API
 Author-email: PolySwarm Developers <info@polyswarm.io>
 License: MIT

{polyswarm_api-4.5.0 → polyswarm_api-4.8.0}/src/polyswarm_api.egg-info/SOURCES.txt

@@ -5,6 +5,7 @@ src/polyswarm_api/__init__.py
 src/polyswarm_api/api.py
 src/polyswarm_api/core.py
 src/polyswarm_api/exceptions.py
+src/polyswarm_api/refang.py
 src/polyswarm_api/resources.py
 src/polyswarm_api/session.py
 src/polyswarm_api/settings.py