TalkPipe 0.4.1__tar.gz → 0.4.2__tar.gz

{talkpipe-0.4.1 → talkpipe-0.4.2}/.github/ISSUE_TEMPLATE/release-checklist.md

@@ -14,6 +14,7 @@ assignees: ''
 - [ ] Build docker image
 - [ ] Tag repository
 - [ ] Build whl files
+- [ ] Install whl file in new environment and test a script in chatterlang_server
 - [ ] Push to pypi
 - [ ] Push code to repo
 - [ ] Push tags to repo

{talkpipe-0.4.1 → talkpipe-0.4.2}/.vscode/settings.json

@@ -1,9 +1,4 @@
 {
-    "python.testing.pytestArgs": [
-        "tests",
-        "-m",
-        "not ddg"
-    ],
     "python.testing.unittestEnabled": false,
     "python.testing.pytestEnabled": true,
     "terminal.integrated.env.osx": {

{talkpipe-0.4.1 → talkpipe-0.4.2}/CHANGELOG.md

@@ -1,6 +1,22 @@
 # Changelog
 
-## 0.4.0
+## 0.4.2 (in progress)
+### New Segments
+ - snippet - lets the script writer either load chatterlang code from a file or provide it as a parameter. 
+ - Abstracted the comparison filter used for gt and lt
+ - Added filters for gte (greater than or equal), lte (less than or equal), eq (equal) and neq (not equal)
+### Improvements
+ - Reconfigured the Docker containers so that there is a base container with the pre-requisites and one for talkpipe.
+ This reduces the time needed to build a talkpipe container if the base already exists.
+ - Retooled unit tests to eliminate the "online" fixture, replacing it with something that checks if ollama and mongo are
+ available locally and skipping tests that require them if they are not available.
+### Bug Fixes
+ - Fixed error in rss.py causing a crash if the URL was specified in the configuration file or environment variable
+ - Handled code where robots.txt response is returned compressed.  It now decompresses it rather than erroring out.
+
+
+
+## 0.4.1
 - Updated syntax so that pipelines in forks can contain more than one segment.  This was an oversight in the original design.
 - Added the mongoInsert and mongoSearch segments
 - Generalized LLM source so that new ones can be registered and used without modifying TalkPipe

{talkpipe-0.4.1 → talkpipe-0.4.2}/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11
+FROM talkpipe:base
 
 WORKDIR /app
 
@@ -17,10 +17,8 @@ COPY .git/ .git/
 
 RUN pip install --upgrade pip
 RUN pip install -e .[dev]
-RUN pytest -m "not online" --log-cli-level=DEBUG
-# RUN pip install -e .
-
-# Install the basics of the scientific computing stack for analytics
-RUN pip install numpy pandas matplotlib scikit-learn scipy
+RUN pytest --log-cli-level=DEBUG
+RUN rm -fr .git
+RUN rm -fr tests
 
 CMD ["python", "-m", "talkpipe.app.runscript", "--load_module", "data/custom_module.py", "--env", "TALKPIPE_SCRIPT"]

{talkpipe-0.4.1 → talkpipe-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: talkpipe
-Version: 0.4.1
+Version: 0.4.2
 Summary: Python internal and external DSL for writing generative AI analytics
 Author-email: Travis Bauer <tlbauer@sandia.gov>
 Maintainer-email: Travis Bauer <tlbauer@sandia.gov>

talkpipe-0.4.2/base-image/Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.11
+
+# Set working directory
+WORKDIR /app
+
+# Install PDM
+RUN pip install pdm
+
+# Copy pyproject.toml into the container
+COPY pyproject.toml ./pyproject.toml
+
+# Run pdm lock, export deps to requirements.txt, and install
+RUN pdm lock && \
+    pdm export --without-hashes --dev -o requirements.txt && \
+    pip install -r requirements.txt && \
+    rm requirements.txt
+
+# Install the basics of the scientific computing stack for analytics
+RUN pip install numpy pandas matplotlib scikit-learn scipy

talkpipe-0.4.2/pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+log_level = DEBUG
+

{talkpipe-0.4.1 → talkpipe-0.4.2}/src/talkpipe/app/static/unit-docs.html

@@ -110,6 +110,11 @@
   <td class="toc-talkpipe">accum</td>
   <td class="toc-description">Accumulates items from the input stream both in an internal buffer and in the specified variable.</td>
 </tr>
+<tr>
+  <td class="toc-name"><a href="#talkpipe-chatterlang-compiler-Snippet">Snippet</a></td>
+  <td class="toc-talkpipe">snippet</td>
+  <td class="toc-description">A segment that loads a chatterlang script from a file and compiles it, after which it</td>
+</tr>
 <tr><td class="toc-package" colspan="3"><a href="#talkpipe-data-email">talkpipe.data.email</a></td></tr>
 <tr>
   <td class="toc-name"><a href="#talkpipe-data-email-sendEmail">sendEmail</a></td>
@@ -357,18 +362,38 @@
 <tr>
   <td class="toc-name"><a href="#talkpipe-pipe-math-arange">arange</a></td>
   <td class="toc-talkpipe">range</td>
-  <td class="toc-description">Generate a range of integers between lower and upper</td>
+  <td class="toc-description">Generate a range of integers between lower (inclusive) and upper (exclusive)</td>
+</tr>
+<tr>
+  <td class="toc-name"><a href="#talkpipe-pipe-math-eq">eq</a></td>
+  <td class="toc-talkpipe">eq</td>
+  <td class="toc-description">Filter items where a specified field&#x27;s value equals a number.</td>
 </tr>
 <tr>
   <td class="toc-name"><a href="#talkpipe-pipe-math-gt">gt</a></td>
   <td class="toc-talkpipe">gt</td>
   <td class="toc-description">Filter items where a specified field&#x27;s value is greater than a number.</td>
 </tr>
+<tr>
+  <td class="toc-name"><a href="#talkpipe-pipe-math-gte">gte</a></td>
+  <td class="toc-talkpipe">gte</td>
+  <td class="toc-description">Filter items where a specified field&#x27;s value is greater than or equal to a number.</td>
+</tr>
 <tr>
   <td class="toc-name"><a href="#talkpipe-pipe-math-lt">lt</a></td>
   <td class="toc-talkpipe">lt</td>
   <td class="toc-description">Filters items based on a field value being less than a specified number.</td>
 </tr>
+<tr>
+  <td class="toc-name"><a href="#talkpipe-pipe-math-lte">lte</a></td>
+  <td class="toc-talkpipe">lte</td>
+  <td class="toc-description">Filter items where a specified field&#x27;s value is less than or equal to a number.</td>
+</tr>
+<tr>
+  <td class="toc-name"><a href="#talkpipe-pipe-math-neq">neq</a></td>
+  <td class="toc-talkpipe">neq</td>
+  <td class="toc-description">Filter items where a specified field&#x27;s value does not equal a number.</td>
+</tr>
 <tr>
   <td class="toc-name"><a href="#talkpipe-pipe-math-randomInts">randomInts</a></td>
   <td class="toc-talkpipe">randomInts</td>
@@ -400,6 +425,22 @@ Args:
   </ul>
   <p><strong>Base Classes:</strong> io.AbstractSegment</p>
 </div>
+<div class="class" id="talkpipe-chatterlang-compiler-Snippet">
+  <h2>Segment Class: Snippet</h2>
+  <p class="talkpipe-name">Chatterlang Name: snippet</p>
+  <div><pre>A segment that loads a chatterlang script from a file and compiles it, after which it
+functions as a normal segment that can be integrated into a pipeline.
+
+Args:
+    file (str): The path to the chatterlang script file.
+    runtime (RuntimeComponent, optional): The runtime component to use. Defaults to None.</pre></div>
+  <h3>Parameters:</h3>
+  <ul class="param-list">
+    <li>self</li>
+    <li>script_source: str</li>
+  </ul>
+  <p><strong>Base Classes:</strong> io.AbstractSegment</p>
+</div>
 <div class="package-header" id="talkpipe-data-email">talkpipe.data.email</div>
 <div class="function" id="talkpipe-data-email-sendEmail">
   <h2>Segment Function: sendEmail</h2>
@@ -1339,68 +1380,182 @@ Args:
 <div class="function" id="talkpipe-pipe-math-arange">
   <h2>Source Function: arange</h2>
   <p class="talkpipe-name">Chatterlang Name: range</p>
-  <div><pre>Generate a range of integers between lower and upper</pre></div>
+  <div><pre>Generate a range of integers between lower (inclusive) and upper (exclusive)
+
+This segment wraps the built-in range function, allowing you to specify
+the lower and upper bounds of the range. The range is inclusive of the
+lower bound and exclusive of the upper bound.
+
+Args:
+    lower (int): Lower bound of the range (inclusive)
+    upper (int): Upper bound of the range (exclusive)</pre></div>
   <h3>Parameters:</h3>
   <ul class="param-list">
     <li>lower</li>
     <li>upper</li>
   </ul>
 </div>
-<div class="function" id="talkpipe-pipe-math-gt">
-  <h2>Segment Function: gt</h2>
+<div class="class" id="talkpipe-pipe-math-eq">
+  <h2>Segment Class: eq</h2>
+  <p class="talkpipe-name">Chatterlang Name: eq</p>
+  <div><pre>Filter items where a specified field's value equals a number.
+
+For each item passed in, this segment yields only those where the value of the specified field
+is equal to the given number n.  
+
+Args:
+    items: Iterable of items to filter
+    field: String representing the field/property to compare.  Note that
+      an underscore "_" can be used to refer to the item itself.
+    n: Item to compare against
+
+Yields:
+    Items where the specified field's value equals n
+
+Raises:
+    AttributeError: If the specified field is missing from any item</pre></div>
+  <h3>Parameters:</h3>
+  <ul class="param-list">
+    <li>self</li>
+    <li>field: str</li>
+    <li>n: Any</li>
+  </ul>
+  <p><strong>Base Classes:</strong> AbstractComparisonFilter</p>
+</div>
+<div class="class" id="talkpipe-pipe-math-gt">
+  <h2>Segment Class: gt</h2>
   <p class="talkpipe-name">Chatterlang Name: gt</p>
   <div><pre>Filter items where a specified field's value is greater than a number.
 
-Takes an iterable of items and yields only those where the value of the specified field
+For each item passed in, this segment yields only those where the value of the specified field
 is greater than the given number n.
 
 Args:
     items: Iterable of items to filter
-    field: String representing the field/property to compare
+    field: String representing the field/property to compare.  Note that
+      an underscore "_" can be used to refer to the item itself.
     n: Number to compare against
 
 Yields:
     Items where the specified field's value is greater than n
 
 Raises:
-    AttributeError: If the specified field is missing from any item
+    AttributeError: If the specified field is missing from any item</pre></div>
+  <h3>Parameters:</h3>
+  <ul class="param-list">
+    <li>self</li>
+    <li>field: str</li>
+    <li>n: Any</li>
+  </ul>
+  <p><strong>Base Classes:</strong> AbstractComparisonFilter</p>
+</div>
+<div class="class" id="talkpipe-pipe-math-gte">
+  <h2>Segment Class: gte</h2>
+  <p class="talkpipe-name">Chatterlang Name: gte</p>
+  <div><pre>Filter items where a specified field's value is greater than or equal to a number.
 
-Example:
-    >>> list(gt([{'value': 5}, {'value': 2}, {'value': 8}], 'value', 4))
-    [{'value': 5}, {'value': 8}]</pre></div>
+For each item passed in, this segment yields only those where the value of the specified field
+is greater than or equal to the given number n.
+
+Args:
+    items: Iterable of items to filter
+    field: String representing the field/property to compare.  Note that
+      an underscore "_" can be used to refer to the item itself.
+    n: Number to compare against
+
+Yields:
+    Items where the specified field's value is greater than or equal to n
+
+Raises:
+    AttributeError: If the specified field is missing from any item</pre></div>
   <h3>Parameters:</h3>
   <ul class="param-list">
-    <li>field</li>
-    <li>n</li>
+    <li>self</li>
+    <li>field: str</li>
+    <li>n: Any</li>
   </ul>
+  <p><strong>Base Classes:</strong> AbstractComparisonFilter</p>
 </div>
-<div class="function" id="talkpipe-pipe-math-lt">
-  <h2>Segment Function: lt</h2>
+<div class="class" id="talkpipe-pipe-math-lt">
+  <h2>Segment Class: lt</h2>
   <p class="talkpipe-name">Chatterlang Name: lt</p>
   <div><pre>Filters items based on a field value being less than a specified number.
 
-Yields items where the specified field value is less than the given number n.
+For each item passed in, this segment yields items where the 
+specified field value is less than the given number n.
 
 Args:
     items (iterable): An iterable of items to filter
-    field (str): The field/property name to check against
+    field: String representing the field/property to compare.  Note that
+      an underscore "_" can be used to refer to the item itself.
     n (numeric): The number to compare against
 
 Yields:
     item: Items where the specified field value is less than n
 
 Raises:
-    AttributeError: If the specified field does not exist on an item (due to fail_on_missing=True)
+    AttributeError: If the specified field does not exist on an item (due to fail_on_missing=True)</pre></div>
+  <h3>Parameters:</h3>
+  <ul class="param-list">
+    <li>self</li>
+    <li>field: str</li>
+    <li>n: Any</li>
+  </ul>
+  <p><strong>Base Classes:</strong> AbstractComparisonFilter</p>
+</div>
+<div class="class" id="talkpipe-pipe-math-lte">
+  <h2>Segment Class: lte</h2>
+  <p class="talkpipe-name">Chatterlang Name: lte</p>
+  <div><pre>Filter items where a specified field's value is less than or equal to a number.
 
-Example:
-    >>> items = [{'val': 1}, {'val': 5}, {'val': 3}]
-    >>> list(lt(items, 'val', 4))
-    [{'val': 1}, {'val': 3}]</pre></div>
+For each item passed in, this segment yields only those where the value of the specified field
+is less than or equal to the given number n.
+
+Args:
+    items: Iterable of items to filter
+    field: String representing the field/property to compare.  Note that
+      an underscore "_" can be used to refer to the item itself.
+    n: Number to compare against
+
+Yields:
+    Items where the specified field's value is less than or equal to n
+
+Raises:
+    AttributeError: If the specified field is missing from any item</pre></div>
   <h3>Parameters:</h3>
   <ul class="param-list">
-    <li>field</li>
-    <li>n</li>
+    <li>self</li>
+    <li>field: str</li>
+    <li>n: Any</li>
+  </ul>
+  <p><strong>Base Classes:</strong> AbstractComparisonFilter</p>
+</div>
+<div class="class" id="talkpipe-pipe-math-neq">
+  <h2>Segment Class: neq</h2>
+  <p class="talkpipe-name">Chatterlang Name: neq</p>
+  <div><pre>Filter items where a specified field's value does not equal a number.
+
+For each item passed in, this segment yields only those where the value of the specified field
+is not equal to the given number n.
+
+Args:
+    items: Iterable of items to filter
+    field: String representing the field/property to compare.  Note that
+      an underscore "_" can be used to refer to the item itself.
+    n: Item to compare against
+
+Yields:
+    Items where the specified field's value does not equal n
+
+Raises:
+    AttributeError: If the specified field is missing from any item</pre></div>
+  <h3>Parameters:</h3>
+  <ul class="param-list">
+    <li>self</li>
+    <li>field: str</li>
+    <li>n: Any</li>
   </ul>
+  <p><strong>Base Classes:</strong> AbstractComparisonFilter</p>
 </div>
 <div class="function" id="talkpipe-pipe-math-randomInts">
   <h2>Source Function: randomInts</h2>

{talkpipe-0.4.1 → talkpipe-0.4.2}/src/talkpipe/app/static/unit-docs.txt

@@ -18,6 +18,19 @@ Segment Class: Accum
     variable: Union[VariableName, str] = None
     reset: bool = True
 
+Segment Class: Snippet
+  Chatterlang Name: snippet
+  Base Classes: io.AbstractSegment
+  Docstring:
+    A segment that loads a chatterlang script from a file and compiles it, after which it
+    functions as a normal segment that can be integrated into a pipeline.
+    
+    Args:
+        file (str): The path to the chatterlang script file.
+        runtime (RuntimeComponent, optional): The runtime component to use. Defaults to None.
+  Parameters:
+    script_source: str
+
 
 PACKAGE: talkpipe.data.email
 ----------------------------
@@ -901,22 +914,56 @@ PACKAGE: talkpipe.pipe.math
 Source Function: arange
   Chatterlang Name: range
   Docstring:
-    Generate a range of integers between lower and upper
+    Generate a range of integers between lower (inclusive) and upper (exclusive)
+    
+    This segment wraps the built-in range function, allowing you to specify
+    the lower and upper bounds of the range. The range is inclusive of the
+    lower bound and exclusive of the upper bound.
+    
+    Args:
+        lower (int): Lower bound of the range (inclusive)
+        upper (int): Upper bound of the range (exclusive)
   Parameters:
     lower
     upper
 
-Segment Function: gt
+Segment Class: eq
+  Chatterlang Name: eq
+  Base Classes: AbstractComparisonFilter
+  Docstring:
+    Filter items where a specified field's value equals a number.
+    
+    For each item passed in, this segment yields only those where the value of the specified field
+    is equal to the given number n.  
+    
+    Args:
+        items: Iterable of items to filter
+        field: String representing the field/property to compare.  Note that
+          an underscore "_" can be used to refer to the item itself.
+        n: Item to compare against
+    
+    Yields:
+        Items where the specified field's value equals n
+    
+    Raises:
+        AttributeError: If the specified field is missing from any item
+  Parameters:
+    field: str
+    n: Any
+
+Segment Class: gt
   Chatterlang Name: gt
+  Base Classes: AbstractComparisonFilter
   Docstring:
     Filter items where a specified field's value is greater than a number.
     
-    Takes an iterable of items and yields only those where the value of the specified field
+    For each item passed in, this segment yields only those where the value of the specified field
     is greater than the given number n.
     
     Args:
         items: Iterable of items to filter
-        field: String representing the field/property to compare
+        field: String representing the field/property to compare.  Note that
+          an underscore "_" can be used to refer to the item itself.
         n: Number to compare against
     
     Yields:
@@ -924,24 +971,47 @@ Segment Function: gt
     
     Raises:
         AttributeError: If the specified field is missing from any item
+  Parameters:
+    field: str
+    n: Any
+
+Segment Class: gte
+  Chatterlang Name: gte
+  Base Classes: AbstractComparisonFilter
+  Docstring:
+    Filter items where a specified field's value is greater than or equal to a number.
     
-    Example:
-        >>> list(gt([{'value': 5}, {'value': 2}, {'value': 8}], 'value', 4))
-        [{'value': 5}, {'value': 8}]
+    For each item passed in, this segment yields only those where the value of the specified field
+    is greater than or equal to the given number n.
+    
+    Args:
+        items: Iterable of items to filter
+        field: String representing the field/property to compare.  Note that
+          an underscore "_" can be used to refer to the item itself.
+        n: Number to compare against
+    
+    Yields:
+        Items where the specified field's value is greater than or equal to n
+    
+    Raises:
+        AttributeError: If the specified field is missing from any item
   Parameters:
-    field
-    n
+    field: str
+    n: Any
 
-Segment Function: lt
+Segment Class: lt
   Chatterlang Name: lt
+  Base Classes: AbstractComparisonFilter
   Docstring:
     Filters items based on a field value being less than a specified number.
     
-    Yields items where the specified field value is less than the given number n.
+    For each item passed in, this segment yields items where the 
+    specified field value is less than the given number n.
     
     Args:
         items (iterable): An iterable of items to filter
-        field (str): The field/property name to check against
+        field: String representing the field/property to compare.  Note that
+          an underscore "_" can be used to refer to the item itself.
         n (numeric): The number to compare against
     
     Yields:
@@ -949,14 +1019,57 @@ Segment Function: lt
     
     Raises:
         AttributeError: If the specified field does not exist on an item (due to fail_on_missing=True)
+  Parameters:
+    field: str
+    n: Any
+
+Segment Class: lte
+  Chatterlang Name: lte
+  Base Classes: AbstractComparisonFilter
+  Docstring:
+    Filter items where a specified field's value is less than or equal to a number.
     
-    Example:
-        >>> items = [{'val': 1}, {'val': 5}, {'val': 3}]
-        >>> list(lt(items, 'val', 4))
-        [{'val': 1}, {'val': 3}]
+    For each item passed in, this segment yields only those where the value of the specified field
+    is less than or equal to the given number n.
+    
+    Args:
+        items: Iterable of items to filter
+        field: String representing the field/property to compare.  Note that
+          an underscore "_" can be used to refer to the item itself.
+        n: Number to compare against
+    
+    Yields:
+        Items where the specified field's value is less than or equal to n
+    
+    Raises:
+        AttributeError: If the specified field is missing from any item
   Parameters:
-    field
-    n
+    field: str
+    n: Any
+
+Segment Class: neq
+  Chatterlang Name: neq
+  Base Classes: AbstractComparisonFilter
+  Docstring:
+    Filter items where a specified field's value does not equal a number.
+    
+    For each item passed in, this segment yields only those where the value of the specified field
+    is not equal to the given number n.
+    
+    Args:
+        items: Iterable of items to filter
+        field: String representing the field/property to compare.  Note that
+          an underscore "_" can be used to refer to the item itself.
+        n: Item to compare against
+    
+    Yields:
+        Items where the specified field's value does not equal n
+    
+    Raises:
+        AttributeError: If the specified field is missing from any item
+  Parameters:
+    field: str
+    n: Any
 
 Source Function: randomInts
   Chatterlang Name: randomInts

{talkpipe-0.4.1 → talkpipe-0.4.2}/src/talkpipe/chatterlang/compiler.py

@@ -33,7 +33,7 @@ def compile(script: ParsedScript, runtime: RuntimeComponent = None) -> Callable:
     logger.debug("Successfully compiled all pipelines")
     return Script(compiled_pipelines)
 
-def _resolve_parame(params, runtime):
+def _resolve_params(params, runtime):
     """ Resolve the parameters for a segment """
     ans = {k: runtime.const_store[params[k].name] if isinstance(params[k], Identifier) else params[k] for k in params}
     return ans
@@ -57,7 +57,7 @@ def _(pipeline: ParsedPipeline, runtime: RuntimeComponent) -> Pipeline:
             ans = VariableSource(pipeline.input_node.source.name)
             logger.debug(f"Created variable source with name {pipeline.input_node.source.name}")
         else:
-            ans = registry.input_registry.get(pipeline.input_node.source.name)(**_resolve_parame(pipeline.input_node.params, runtime=runtime))
+            ans = registry.input_registry.get(pipeline.input_node.source.name)(**_resolve_params(pipeline.input_node.params, runtime=runtime))
             logger.debug(f"Created registered input {pipeline.input_node.source.name}")
         ans.runtime = runtime
 
@@ -67,7 +67,7 @@ def _(pipeline: ParsedPipeline, runtime: RuntimeComponent) -> Pipeline:
             next_transform = VariableSetSegment(transform.name)
             logger.debug(f"Created variable set segment for {transform.name}")
         elif isinstance(transform, SegmentNode):
-            next_transform = registry.segment_registry.get(transform.operation.name)(**_resolve_parame(transform.params, runtime=runtime))
+            next_transform = registry.segment_registry.get(transform.operation.name)(**_resolve_params(transform.params, runtime=runtime))
             logger.debug(f"Created segment {transform.operation.name}")
         elif isinstance(transform, ForkNode):
             next_transform = compile(transform, runtime)
@@ -216,3 +216,30 @@ class Accum(io.AbstractSegment):
             if self.variable_name:
                 self.runtime.variable_store[self.variable_name].append(item)
             yield item
+
+
+@registry.register_segment(name="snippet")
+class Snippet(io.AbstractSegment):
+    """A segment that loads a chatterlang script from a file and compiles it, after which it
+    functions as a normal segment that can be integrated into a pipeline.
+
+    Args:
+        file (str): The path to the chatterlang script file.
+        runtime (RuntimeComponent, optional): The runtime component to use. Defaults to None.
+    """    
+
+    def __init__(self, script_source: str):
+        super().__init__()
+        self.script_source = script_source
+        self.script = None
+
+    def transform(self, items):
+        if self.script is None:
+            try:
+                with open(self.script_source, "r") as f:
+                    script_text = f.read()
+            except FileNotFoundError:
+                # If file doesn't exist, treat self.script_source as the script content
+                script_text = self.script_source
+            self.script = compile(script_text, self.runtime)
+        yield from self.script(items)

{talkpipe-0.4.1 → talkpipe-0.4.2}/src/talkpipe/data/html.py

@@ -2,6 +2,7 @@
 
 import logging
 import re
+import gzip
 import urllib.error
 import requests
 import urllib
@@ -97,7 +98,13 @@ def get_robot_parser(domain, timeout=5):
 
     try:
         with urllib.request.urlopen(robots_url, timeout=timeout) as response:
-            content = response.read().decode('utf-8')
+            content = response.read()
+            if content.startswith(b'\x1f\x8b'):
+                # If the content is gzipped, decompress it
+                content = gzip.decompress(content).decode('utf-8')
+            else:
+                # If not gzipped, decode it directly
+                content = content.decode('utf-8')
         
         try:
             rp.parse(content.splitlines())