sqlframe 1.11.0__py3-none-any.whl → 1.12.0__py3-none-any.whl

sqlframe/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.11.0'
-__version_tuple__ = version_tuple = (1, 11, 0)
+__version__ = version = '1.12.0'
+__version_tuple__ = version_tuple = (1, 12, 0)

sqlframe/base/column.py

@@ -407,3 +407,44 @@ class Column:
         window_expression = window.expression.copy()
         window_expression.set("this", self.column_expression)
         return Column(window_expression)
+
+    def getItem(self, key: t.Any) -> Column:
+        """
+        An expression that gets an item at position ``ordinal`` out of a list,
+        or gets an item by key out of a dict.
+
+        .. versionadded:: 1.3.0
+
+        .. versionchanged:: 3.4.0
+            Supports Spark Connect.
+
+        Parameters
+        ----------
+        key
+            a literal value, or a :class:`Column` expression.
+            The result will only be true at a location if the item matches in the column.
+
+             .. deprecated:: 3.0.0
+                 :class:`Column` as a parameter is deprecated.
+
+        Returns
+        -------
+        :class:`Column`
+            Column representing the item(s) got at position out of a list or by key out of a dict.
+
+        Examples
+        --------
+        >>> df = spark.createDataFrame([([1, 2], {"key": "value"})], ["l", "d"])
+        >>> df.select(df.l.getItem(0), df.d.getItem("key")).show()
+        +----+------+
+        |l[0]|d[key]|
+        +----+------+
+        |   1| value|
+        +----+------+
+        """
+        element_at = get_func_from_session("element_at")
+        lit = get_func_from_session("lit")
+        key = lit(key) if not isinstance(key, Column) else key
+        if isinstance(key.expression, exp.Literal) and key.expression.is_number:
+            key = key + lit(1)
+        return element_at(self, key)

sqlframe/base/dataframe.py

@@ -361,7 +361,7 @@ class _BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
 
         cols = self._ensure_list_of_columns(cols)
         normalize(self.session, expression or self.expression, cols)
-        return cols
+        return list(flatten([self._expand_star(col) for col in cols]))
 
     def _ensure_and_normalize_col(self, col):
         from sqlframe.base.column import Column
@@ -514,6 +514,27 @@ class _BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
         select_expressions.append(expression_select_pair)  # type: ignore
         return select_expressions
 
+    def _expand_star(self, col: Column) -> t.List[Column]:
+        from sqlframe.base.column import Column
+
+        if isinstance(col.column_expression, exp.Star):
+            return self._get_outer_select_columns(self.expression)
+        elif (
+            isinstance(col.column_expression, exp.Column)
+            and isinstance(col.column_expression.this, exp.Star)
+            and col.column_expression.args.get("table")
+        ):
+            for cte in self.expression.ctes:
+                if cte.alias_or_name == col.column_expression.args["table"].this:
+                    return [
+                        Column.ensure_col(exp.column(x.column_alias_or_name, cte.alias_or_name))
+                        for x in self._get_outer_select_columns(cte)
+                    ]
+            raise ValueError(
+                f"Could not find table to expand star: {col.column_expression.args['table']}"
+            )
+        return [col]
+
     @t.overload
     def sql(
         self,
@@ -1555,7 +1576,7 @@ class _BaseDataFrame(t.Generic[SESSION, WRITER, NA, STAT, GROUP_DATA]):
             result = self.session._fetch_rows(sql)
         table = PrettyTable()
         if row := seq_get(result, 0):
-            table.field_names = list(row.asDict().keys())
+            table.field_names = row._unique_field_names
             for row in result:
                 table.add_row(list(row))
         print(table)

sqlframe/base/function_alternatives.py

@@ -1135,13 +1135,11 @@ def array_intersect_using_intersection(col1: ColumnOrName, col2: ColumnOrName) -
 def element_at_using_brackets(col: ColumnOrName, value: ColumnOrLiteral) -> Column:
     col_func = get_func_from_session("col")
     lit = get_func_from_session("lit")
-    #  SQLGlot will auto add 1 to whatever we pass in for the brackets even though the value is already 1 based.
-    if not isinstance(value, int):
-        raise ValueError("This dialect requires the value must be an integer")
-    value_lit = lit(value - 1)
-    return Column(
-        expression.Bracket(this=col_func(col).expression, expressions=[value_lit.expression])
-    )
+    # SQLGlot will auto add 1 to whatever we pass in for the brackets even though the value is already 1 based.
+    value = value if isinstance(value, Column) else lit(value)
+    if [x for x in value.expression.find_all(expression.Literal) if x.is_number]:
+        value = value - lit(1)
+    return Column(expression.Bracket(this=col_func(col).expression, expressions=[value.expression]))  # type: ignore
 
 
 def array_remove_using_filter(col: ColumnOrName, value: ColumnOrLiteral) -> Column:

sqlframe/base/functions.py

@@ -1923,7 +1923,9 @@ def call_function(funcName: str, *cols: ColumnOrName) -> Column:
     cols = ensure_list(cols)  # type: ignore
     if len(cols) > 1:
         return Column.invoke_anonymous_function(cols[0], funcName, *cols[1:])
-    return Column.invoke_anonymous_function(cols[0], funcName)
+    elif len(cols) == 1:
+        return Column.invoke_anonymous_function(cols[0], funcName)
+    return Column.invoke_anonymous_function(None, funcName)
 
 
 # @meta(unsupported_engines="*")
@@ -2028,7 +2030,7 @@ def character_length(str: ColumnOrName) -> Column:
     return Column.invoke_anonymous_function(str, "character_length")
 
 
-@meta(unsupported_engines="*")
+@meta()
 def contains(left: ColumnOrName, right: ColumnOrName) -> Column:
     return Column.invoke_anonymous_function(left, "contains", right)
 

sqlframe/base/types.py

@@ -416,3 +416,13 @@ class Row(tuple):
             )
         else:
             return "<Row(%s)>" % ", ".join(repr(field) for field in self)
+
+    # SQLFrame Specific
+    @property
+    def _unique_field_names(self) -> t.List[str]:
+        fields = []
+        for i, field in enumerate(self.__fields__):
+            if field in fields:
+                field = field + "_" + str(i)
+            fields.append(field)
+        return fields

sqlframe/bigquery/functions.py

@@ -52,6 +52,7 @@ from sqlframe.base.function_alternatives import (  # noqa
     make_date_from_date_func as make_date,
     to_date_from_timestamp as to_date,
     last_day_with_cast as last_day,
+    sha1_force_sha1_and_to_hex as sha,
     sha1_force_sha1_and_to_hex as sha1,
     hash_from_farm_fingerprint as hash,
     base64_from_blob as base64,

{sqlframe-1.11.0.dist-info → sqlframe-1.12.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sqlframe
-Version: 1.11.0
+Version: 1.12.0
 Summary: Turning PySpark Into a Universal DataFrame API
 Home-page: https://github.com/eakmanrq/sqlframe
 Author: Ryan Eakman

{sqlframe-1.11.0.dist-info → sqlframe-1.12.0.dist-info}/RECORD

@@ -1,21 +1,21 @@
 sqlframe/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sqlframe/_version.py,sha256=rZqhcUFwPMyj_mTWUN2A6qcFr8Ptv08CSbXbruC3jR4,413
+sqlframe/_version.py,sha256=cgR9Mx-45EIoPNcTiuL_LLmCR2oVTYIPi5z0W11owvc,413
 sqlframe/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sqlframe/base/_typing.py,sha256=DuTay8-o9W-pw3RPZCgLunKNJLS9PkaV11G_pxXp9NY,1256
 sqlframe/base/catalog.py,sha256=ATDGirouUjal05P4ymL-wIi8rgjg_8w4PoACamiO64A,37245
-sqlframe/base/column.py,sha256=5bfJWj9dnStHUxLSrWMD-gwiC4-aHKC8lhoC62nhM1k,16153
-sqlframe/base/dataframe.py,sha256=Tf5euWTGxFmYirgHK5ZXUI41so5ruo-asVmUwj9DFdo,70015
+sqlframe/base/column.py,sha256=y41rFV7y_seTNkAK3SSqnggGi2otXt0ejKzsMyHCYT4,17515
+sqlframe/base/dataframe.py,sha256=75ZM9r52fufFmVShtntcDUr6dZ1stX9HDmXLuDrYTAU,71004
 sqlframe/base/decorators.py,sha256=I5osMgx9BuCgbtp4jVM2DNwYJVLzCv-OtTedhQEik0g,1882
 sqlframe/base/exceptions.py,sha256=9Uwvqn2eAkDpqm4BrRgbL61qM-GMCbJEMAW8otxO46s,370
-sqlframe/base/function_alternatives.py,sha256=QESqZy7Osp9-CV5Yoi6XFat5SE8PzCVZ3o7gOFmIY7g,45888
-sqlframe/base/functions.py,sha256=hJDpE7GYQpQ1iHjdr1hG_hu0mAIb60vNoghjEcgMREI,187550
+sqlframe/base/function_alternatives.py,sha256=l6Fu0mZ-eleObpYcCAnOXV1HvuHugeoCFUcSV7NmFis,45916
+sqlframe/base/functions.py,sha256=hSLuyO03m2dXPJdmVKp9of-_xj4V2sUhSzJ65Ti240Q,187616
 sqlframe/base/group.py,sha256=TES9CleVmH3x-0X-tqmuUKfCKSWjH5vg1aU3R6dDmFc,4059
 sqlframe/base/normalize.py,sha256=nXAJ5CwxVf4DV0GsH-q1w0p8gmjSMlv96k_ez1eVul8,3880
 sqlframe/base/operations.py,sha256=-AhNuEzcV7ZExoP1oY3blaKip-joQyJeQVvfBTs_2g4,3456
 sqlframe/base/readerwriter.py,sha256=5NPQMiOrw6I54U243R_6-ynnWYsNksgqwRpPp4IFjIw,25288
 sqlframe/base/session.py,sha256=2C0OsPoP49AuqVNtPiazTdVpwQA1668g5WOydrYP6SA,22001
 sqlframe/base/transforms.py,sha256=y0j3SGDz3XCmNGrvassk1S-owllUWfkHyMgZlY6SFO4,467
-sqlframe/base/types.py,sha256=K6mjafbX7oIk65CapwamcO2I8nf-poRIpKKt9XDNEaQ,11987
+sqlframe/base/types.py,sha256=iBNk9bpFtb2NBIogYS8i7OlQZMRvpR6XxqzBebsjQDU,12280
 sqlframe/base/util.py,sha256=tWccrZne-Acn4N2RxYr87mfI_GDMf_K9hRD7BnhGBq0,11756
 sqlframe/base/window.py,sha256=8hOv-ignPPIsZA9FzvYzcLE9J_glalVaYjIAUdRUX3o,4943
 sqlframe/base/mixins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -26,7 +26,7 @@ sqlframe/bigquery/__init__.py,sha256=i2NsMbiXOj2xphCtPuNk6cVw4iYeq5_B1I9dVI9aGAk
 sqlframe/bigquery/catalog.py,sha256=h3aQAQAJg6MMvFpP8Ku0S4pcx30n5qYrqHhWSomxb6A,9319
 sqlframe/bigquery/column.py,sha256=E1tUa62Y5HajkhgFuebU9zohrGyieudcHzTT8gfalio,40
 sqlframe/bigquery/dataframe.py,sha256=Y2uy4FEYw0KxIHgnaA9uMwdIzxJzTlD_NSzIe7P7kxA,2405
-sqlframe/bigquery/functions.py,sha256=2YqJmBG0F0o10cztFZoP-G4px1QMKuHST6jlj1snUfY,11331
+sqlframe/bigquery/functions.py,sha256=ifJxEyHDwSp2iA-yBt7XHLh9GhVPpgzs5YckMFvag8w,11370
 sqlframe/bigquery/functions.pyi,sha256=JiyLFLiO0jyJec6j1o4ujPVQ7Tma-c9YHlm-3UQYD9M,13642
 sqlframe/bigquery/group.py,sha256=UVBNBRTo8OqS-_cS5YwvTeJYgYxeG-d6R3kfyHmlFqw,391
 sqlframe/bigquery/readwriter.py,sha256=WAD3ZMwkkjOpvPPoZXfaLLNM6tRTeUvdEj-hQZAzXeo,870
@@ -99,8 +99,8 @@ sqlframe/standalone/types.py,sha256=KwNyuXIo-2xVVd4bZED3YrQOobKCtemlxGrJL7DrTC8,
 sqlframe/standalone/window.py,sha256=6GKPzuxeSapJakBaKBeT9VpED1ACdjggDv9JRILDyV0,35
 sqlframe/testing/__init__.py,sha256=VVCosQhitU74A3NnE52O4mNtGZONapuEXcc20QmSlnQ,132
 sqlframe/testing/utils.py,sha256=9DDYVuocO7tygee3RaajuJNZ24sJwf_LY556kKg7kTw,13011
-sqlframe-1.11.0.dist-info/LICENSE,sha256=VZu79YgW780qxaFJMr0t5ZgbOYEh04xWoxaWOaqIGWk,1068
-sqlframe-1.11.0.dist-info/METADATA,sha256=JTMUu99Ygcz_fK15KTHUb9OqQcPiQoUjQ1-7RQ09COE,7497
-sqlframe-1.11.0.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-sqlframe-1.11.0.dist-info/top_level.txt,sha256=T0_RpoygaZSF6heeWwIDQgaP0varUdSK1pzjeJZRjM8,9
-sqlframe-1.11.0.dist-info/RECORD,,
+sqlframe-1.12.0.dist-info/LICENSE,sha256=VZu79YgW780qxaFJMr0t5ZgbOYEh04xWoxaWOaqIGWk,1068
+sqlframe-1.12.0.dist-info/METADATA,sha256=pSBFDDNjWo6RscllTU6EquCE1DG0C8FXmuad1RINxgk,7497
+sqlframe-1.12.0.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+sqlframe-1.12.0.dist-info/top_level.txt,sha256=T0_RpoygaZSF6heeWwIDQgaP0varUdSK1pzjeJZRjM8,9
+sqlframe-1.12.0.dist-info/RECORD,,