ingestr 0.12.5__tar.gz → 0.12.6__tar.gz

{ingestr-0.12.5 → ingestr-0.12.6}/Makefile

@@ -24,7 +24,7 @@ test-specific: venv
 	. venv/bin/activate; pytest -rP -vv --tb=short --capture=no -k $(test)
 
 lint-ci:
-	ruff check ingestr --fix && ruff format ingestr
+	ruff format ingestr && ruff check ingestr --fix
 	mypy --config-file pyproject.toml --explicit-package-bases ingestr
 
 lint: venv

{ingestr-0.12.5 → ingestr-0.12.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.12.5
+Version: 0.12.6
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -26,6 +26,7 @@ Requires-Dist: google-api-python-client==2.130.0
 Requires-Dist: google-cloud-bigquery-storage==2.24.0
 Requires-Dist: mysql-connector-python==9.1.0
 Requires-Dist: pendulum==3.0.0
+Requires-Dist: psutil==6.1.1
 Requires-Dist: psycopg2-binary==2.9.10
 Requires-Dist: py-machineid==0.6.0
 Requires-Dist: pyairtable==2.3.3

{ingestr-0.12.5 → ingestr-0.12.6}/docs/supported-sources/github.md

@@ -14,9 +14,9 @@ github://?access_token=<access_token>&owner=<owner>&repo=<repo>
 
 URI parameters:
 
-- `access_token`: Access Token used for authentication with the GitHub API
-- `owner`: Refers to the owner of the repository
-- `repo`: Refers to the name of the repository
+- `access_token` (optional): Access Token used for authentication with the GitHub API
+- `owner` (required): Refers to the owner of the repository
+- `repo` (required): Refers to the name of the repository
 
 
 ## Setting up a GitHub Integration

{ingestr-0.12.5 → ingestr-0.12.6}/ingestr/main.py

@@ -444,7 +444,7 @@ def ingest(
 
         progressInstance: Collector = SpinnerCollector()
         if progress == Progress.log:
-            progressInstance = LogCollector(dump_system_stats=False)
+            progressInstance = LogCollector()
 
         is_pipelines_dir_temp = False
         if pipelines_dir is None:

{ingestr-0.12.5 → ingestr-0.12.6}/ingestr/src/asana_source/__init__.py

@@ -150,7 +150,10 @@ def tasks(
     project_array: t.List[TDataItem],
     access_token: str = dlt.secrets.value,
     modified_at: dlt.sources.incremental[str] = dlt.sources.incremental(
-        "modified_at", initial_value=DEFAULT_START_DATE
+        "modified_at",
+        initial_value=DEFAULT_START_DATE,
+        range_end="closed",
+        range_start="closed",
     ),
     fields: Iterable[str] = TASK_FIELDS,
 ) -> Iterable[TDataItem]:

{ingestr-0.12.5 → ingestr-0.12.6}/ingestr/src/facebook_ads/__init__.py

@@ -159,7 +159,10 @@ def facebook_insights_source(
     )
     def facebook_insights(
         date_start: dlt.sources.incremental[str] = dlt.sources.incremental(
-            "date_start", initial_value=initial_load_start_date_str
+            "date_start",
+            initial_value=initial_load_start_date_str,
+            range_end="closed",
+            range_start="closed",
         ),
     ) -> Iterator[TDataItems]:
         start_date = get_start_date(date_start, attribution_window_days_lag)

{ingestr-0.12.5 → ingestr-0.12.6}/ingestr/src/filesystem/__init__.py

@@ -38,7 +38,9 @@ def readers(
     """
     filesystem_resource = filesystem(bucket_url, credentials, file_glob=file_glob)
     filesystem_resource.apply_hints(
-        incremental=dlt.sources.incremental("modification_date")
+        incremental=dlt.sources.incremental("modification_date"),
+        range_end="closed",
+        range_start="closed",
     )
     return (
         filesystem_resource | dlt.transformer(name="read_csv")(_read_csv),

{ingestr-0.12.5 → ingestr-0.12.6}/ingestr/src/github/__init__.py

@@ -14,7 +14,7 @@ from .helpers import get_reactions_data, get_rest_pages, get_stargazers
 def github_reactions(
     owner: str,
     name: str,
-    access_token: str = dlt.secrets.value,
+    access_token: str,
     items_per_page: int = 100,
     max_items: Optional[int] = None,
 ) -> Sequence[DltResource]:
@@ -89,7 +89,11 @@ def github_repo_events(
     @dlt.resource(primary_key="id", table_name=lambda i: i["type"])
     def repo_events(
         last_created_at: dlt.sources.incremental[str] = dlt.sources.incremental(
-            "created_at", initial_value="1970-01-01T00:00:00Z", last_value_func=max
+            "created_at",
+            initial_value="1970-01-01T00:00:00Z",
+            last_value_func=max,
+            range_end="closed",
+            range_start="closed",
         ),
     ) -> Iterator[TDataItems]:
         repos_path = (
@@ -114,7 +118,7 @@ def github_repo_events(
 def github_stargazers(
     owner: str,
     name: str,
-    access_token: str = dlt.secrets.value,
+    access_token: str,
     items_per_page: int = 100,
     max_items: Optional[int] = None,
 ) -> Sequence[DltResource]:

ingestr-0.12.6/ingestr/src/google_analytics/__init__.py

@@ -0,0 +1,106 @@
+"""
+Defines all the sources and resources needed for Google Analytics V4
+"""
+
+from typing import Iterator, List, Optional, Union
+
+import dlt
+from dlt.common import pendulum
+from dlt.common.typing import DictStrAny, TDataItem
+from dlt.extract import DltResource
+from dlt.sources.credentials import GcpOAuthCredentials, GcpServiceAccountCredentials
+from google.analytics.data_v1beta import BetaAnalyticsDataClient
+from google.analytics.data_v1beta.types import (
+    Dimension,
+    Metric,
+)
+
+from .helpers import get_report
+
+
+@dlt.source(max_table_nesting=0)
+def google_analytics(
+    datetime_dimension: str,
+    credentials: Union[
+        GcpOAuthCredentials, GcpServiceAccountCredentials
+    ] = dlt.secrets.value,
+    property_id: int = dlt.config.value,
+    queries: List[DictStrAny] = dlt.config.value,
+    start_date: Optional[pendulum.DateTime] = pendulum.datetime(2024, 1, 1),
+    end_date: Optional[pendulum.DateTime] = None,
+    rows_per_page: int = 10000,
+) -> List[DltResource]:
+    try:
+        property_id = int(property_id)
+    except ValueError:
+        raise ValueError(
+            f"{property_id} is an invalid google property id. Please use a numeric id, and not your Measurement ID like G-7F1AE12JLR"
+        )
+    if property_id == 0:
+        raise ValueError(
+            "Google Analytics property id is 0. Did you forget to configure it?"
+        )
+    if not rows_per_page:
+        raise ValueError("Rows per page cannot be 0")
+    # generate access token for credentials if we are using OAuth2.0
+    if isinstance(credentials, GcpOAuthCredentials):
+        credentials.auth("https://www.googleapis.com/auth/analytics.readonly")
+
+    # Build the service object for Google Analytics api.
+    client = BetaAnalyticsDataClient(credentials=credentials.to_native_credentials())
+    if len(queries) > 1:
+        raise ValueError(
+            "Google Analytics supports a single query ingestion at a time, please give only one query"
+        )
+    query = queries[0]
+
+    # always add "date" to dimensions so we are able to track the last day of a report
+    dimensions = query["dimensions"]
+
+    @dlt.resource(
+        name="basic_report",
+        merge_key=datetime_dimension,
+        write_disposition="merge",
+    )
+    def basic_report(
+        incremental=dlt.sources.incremental(
+            datetime_dimension,
+            initial_value=start_date,
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterator[TDataItem]:
+        start_date = incremental.last_value
+        end_date = incremental.end_value
+        if start_date is None:
+            start_date = pendulum.datetime(2024, 1, 1)
+        if end_date is None:
+            end_date = pendulum.yesterday()
+        yield from get_report(
+            client=client,
+            property_id=property_id,
+            dimension_list=[Dimension(name=dimension) for dimension in dimensions],
+            metric_list=[Metric(name=metric) for metric in query["metrics"]],
+            per_page=rows_per_page,
+            start_date=start_date,
+            end_date=end_date,
+        )
+
+    # res = dlt.resource(
+    #     basic_report, name="basic_report", merge_key=datetime_dimension, write_disposition="merge"
+    # )(
+    #     client=client,
+    #     rows_per_page=rows_per_page,
+    #     property_id=property_id,
+    #     dimensions=dimensions,
+    #     metrics=query["metrics"],
+    #     resource_name=resource_name,
+    #     last_date=dlt.sources.incremental(
+    #         datetime_dimension,
+    #         initial_value=start_date,
+    #         end_value=end_date,
+    #     ),
+    # )
+
+    return [basic_report]

ingestr-0.12.5/ingestr/src/google_analytics/helpers/data_processing.py → ingestr-0.12.6/ingestr/src/google_analytics/helpers.py

@@ -57,9 +57,9 @@ def get_report(
     property_id: int,
     dimension_list: List[Dimension],
     metric_list: List[Metric],
-    limit: int,
-    start_date: str,
-    end_date: str,
+    per_page: int,
+    start_date: pendulum.DateTime,
+    end_date: pendulum.DateTime,
 ) -> Iterator[TDataItem]:
     """
     Gets all the possible pages of reports with the given query parameters.
@@ -79,30 +79,36 @@ def get_report(
         Generator of all rows of data in the report.
     """
 
-    request = RunReportRequest(
-        property=f"properties/{property_id}",
-        dimensions=dimension_list,
-        metrics=metric_list,
-        limit=limit,
-        date_ranges=[DateRange(start_date=start_date, end_date=end_date)],
+    print(
+        "fetching for daterange", start_date.to_date_string(), end_date.to_date_string()
     )
-    # process request
-    response = client.run_report(request)
-    processed_response_generator = process_report(response=response)
-    yield from processed_response_generator
 
+    offset = 0
+    while True:
+        request = RunReportRequest(
+            property=f"properties/{property_id}",
+            dimensions=dimension_list,
+            metrics=metric_list,
+            limit=per_page,
+            offset=offset,
+            date_ranges=[
+                DateRange(
+                    start_date=start_date.to_date_string(),
+                    end_date=end_date.to_date_string(),
+                )
+            ],
+        )
+        # process request
+        response = client.run_report(request)
+        processed_response_generator = process_report(response=response)
+        # import pdb; pdb.set_trace()
+        yield from processed_response_generator
+        offset += per_page
+        if len(response.rows) < per_page or offset > 1000000:
+            break
 
-def process_report(response: RunReportResponse) -> Iterator[TDataItems]:
-    """
-    Receives a single page for a report response, processes it, and returns a generator for every row of data in the report page.
-
-    Args:
-        response: The API response for a single page of the report.
-
-    Yields:
-        Generator of dictionaries for every row of the report page.
-    """
 
+def process_report(response: RunReportResponse) -> Iterator[TDataItems]:
     metrics_headers = [header.name for header in response.metric_headers]
     dimensions_headers = [header.name for header in response.dimension_headers]
 
@@ -156,16 +162,6 @@ def process_metric_value(metric_type: MetricType, value: str) -> Union[str, int,
 
 
 def _resolve_dimension_value(dimension_name: str, dimension_value: str) -> Any:
-    """
-    Helper function that receives a dimension's name and value and converts it to a datetime object if needed.
-
-    Args:
-        dimension_name: Name of the dimension.
-        dimension_value: Value of the dimension.
-
-    Returns:
-        The value of the dimension with the correct data type.
-    """
     if dimension_name == "date":
         return pendulum.from_format(dimension_value, "YYYYMMDD", tz="UTC")
     elif dimension_name == "dateHour":

{ingestr-0.12.5 → ingestr-0.12.6}/ingestr/src/gorgias/__init__.py

@@ -116,7 +116,9 @@ def gorgias_source(
         },
     )
     def customers(
-        updated_datetime=dlt.sources.incremental("updated_datetime", start_date_obj),
+        updated_datetime=dlt.sources.incremental(
+            "updated_datetime", start_date_obj, range_end="closed", range_start="closed"
+        ),
     ) -> Iterable[TDataItem]:
         """
         The resource for customers on your Gorgias domain, supports incremental loading and pagination.
@@ -290,7 +292,9 @@ def gorgias_source(
         },
     )
     def tickets(
-        updated_datetime=dlt.sources.incremental("updated_datetime", start_date_obj),
+        updated_datetime=dlt.sources.incremental(
+            "updated_datetime", start_date_obj, range_end="closed", range_start="closed"
+        ),
     ) -> Iterable[TDataItem]:
         """
         The resource for tickets on your Gorgias domain, supports incremental loading and pagination.
@@ -481,7 +485,9 @@ def gorgias_source(
         },
     )
     def ticket_messages(
-        updated_datetime=dlt.sources.incremental("updated_datetime", start_date_obj),
+        updated_datetime=dlt.sources.incremental(
+            "updated_datetime", start_date_obj, range_end="closed", range_start="closed"
+        ),
     ) -> Iterable[TDataItem]:
         """
         The resource for ticket messages on your Gorgias domain, supports incremental loading and pagination.
@@ -566,7 +572,9 @@ def gorgias_source(
         },
     )
     def satisfaction_surveys(
-        updated_datetime=dlt.sources.incremental("updated_datetime", start_date_obj),
+        updated_datetime=dlt.sources.incremental(
+            "updated_datetime", start_date_obj, range_end="closed", range_start="closed"
+        ),
     ) -> Iterable[TDataItem]:
         """
         The resource for satisfaction surveys on your Gorgias domain, supports incremental loading and pagination.

{ingestr-0.12.5 → ingestr-0.12.6}/ingestr/src/hubspot/__init__.py

@@ -278,4 +278,11 @@ def hubspot_events_for_objects(
         write_disposition="append",
         selected=True,
         table_name=lambda e: name + "_" + str(e["eventType"]),
-    )(dlt.sources.incremental("occurredAt", initial_value=start_date.isoformat()))
+    )(
+        dlt.sources.incremental(
+            "occurredAt",
+            initial_value=start_date.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        )
+    )

{ingestr-0.12.5 → ingestr-0.12.6}/ingestr/src/klaviyo/_init_.py

@@ -33,7 +33,12 @@ def klaviyo_source(api_key: str, start_date: TAnyDateTime) -> Iterable[DltResour
 
     @dlt.resource(write_disposition="append", primary_key="id", parallelized=True)
     def events(
-        datetime=dlt.sources.incremental("datetime", start_date_obj.isoformat()),
+        datetime=dlt.sources.incremental(
+            "datetime",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
     ) -> Iterable[TDataItem]:
         intervals = split_date_range(
             pendulum.parse(datetime.start_value), pendulum.now()
@@ -44,7 +49,12 @@ def klaviyo_source(api_key: str, start_date: TAnyDateTime) -> Iterable[DltResour
 
     @dlt.resource(write_disposition="merge", primary_key="id", parallelized=True)
     def profiles(
-        updated=dlt.sources.incremental("updated", start_date_obj.isoformat()),
+        updated=dlt.sources.incremental(
+            "updated",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
     ) -> Iterable[TDataItem]:
         intervals = split_date_range(
             pendulum.parse(updated.start_value), pendulum.now()
@@ -55,7 +65,12 @@ def klaviyo_source(api_key: str, start_date: TAnyDateTime) -> Iterable[DltResour
 
     @dlt.resource(write_disposition="merge", primary_key="id", parallelized=True)
     def campaigns(
-        updated_at=dlt.sources.incremental("updated_at", start_date_obj.isoformat()),
+        updated_at=dlt.sources.incremental(
+            "updated_at",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
     ) -> Iterable[TDataItem]:
         intervals = split_date_range(
             pendulum.parse(updated_at.start_value), pendulum.now()
@@ -69,7 +84,12 @@ def klaviyo_source(api_key: str, start_date: TAnyDateTime) -> Iterable[DltResour
 
     @dlt.resource(write_disposition="merge", primary_key="id")
     def metrics(
-        updated=dlt.sources.incremental("updated", start_date_obj.isoformat()),
+        updated=dlt.sources.incremental(
+            "updated",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
     ) -> Iterable[TDataItem]:
         yield from client.fetch_metrics(create_client(), updated.start_value)
 
@@ -83,7 +103,12 @@ def klaviyo_source(api_key: str, start_date: TAnyDateTime) -> Iterable[DltResour
 
     @dlt.resource(write_disposition="merge", primary_key="id", name="catalog-variants")
     def catalog_variants(
-        updated=dlt.sources.incremental("updated", start_date_obj.isoformat()),
+        updated=dlt.sources.incremental(
+            "updated",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
     ) -> Iterable[TDataItem]:
         yield from client.fetch_catalog_variant(create_client(), updated.start_value)
 
@@ -91,19 +116,34 @@ def klaviyo_source(api_key: str, start_date: TAnyDateTime) -> Iterable[DltResour
         write_disposition="merge", primary_key="id", name="catalog-categories"
     )
     def catalog_categories(
-        updated=dlt.sources.incremental("updated", start_date_obj.isoformat()),
+        updated=dlt.sources.incremental(
+            "updated",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
     ) -> Iterable[TDataItem]:
         yield from client.fetch_catalog_categories(create_client(), updated.start_value)
 
     @dlt.resource(write_disposition="merge", primary_key="id", name="catalog-items")
     def catalog_items(
-        updated=dlt.sources.incremental("updated", start_date_obj.isoformat()),
+        updated=dlt.sources.incremental(
+            "updated",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
     ) -> Iterable[TDataItem]:
         yield from client.fetch_catalog_item(create_client(), updated.start_value)
 
     @dlt.resource(write_disposition="append", primary_key="id", parallelized=True)
     def forms(
-        updated_at=dlt.sources.incremental("updated_at", start_date_obj.isoformat()),
+        updated_at=dlt.sources.incremental(
+            "updated_at",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
     ) -> Iterable[TDataItem]:
         intervals = split_date_range(
             pendulum.parse(updated_at.start_value), pendulum.now()
@@ -114,13 +154,23 @@ def klaviyo_source(api_key: str, start_date: TAnyDateTime) -> Iterable[DltResour
 
     @dlt.resource(write_disposition="merge", primary_key="id")
     def lists(
-        updated=dlt.sources.incremental("updated", start_date_obj.isoformat()),
+        updated=dlt.sources.incremental(
+            "updated",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
     ) -> Iterable[TDataItem]:
         yield from client.fetch_lists(create_client(), updated.start_value)
 
     @dlt.resource(write_disposition="append", primary_key="id", parallelized=True)
     def images(
-        updated_at=dlt.sources.incremental("updated_at", start_date_obj.isoformat()),
+        updated_at=dlt.sources.incremental(
+            "updated_at",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
     ) -> Iterable[TDataItem]:
         intervals = split_date_range(
             pendulum.parse(updated_at.start_value), pendulum.now()
@@ -130,13 +180,23 @@ def klaviyo_source(api_key: str, start_date: TAnyDateTime) -> Iterable[DltResour
 
     @dlt.resource(write_disposition="merge", primary_key="id")
     def segments(
-        updated=dlt.sources.incremental("updated", start_date_obj.isoformat()),
+        updated=dlt.sources.incremental(
+            "updated",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
     ) -> Iterable[TDataItem]:
         yield from client.fetch_segments(create_client(), updated.start_value)
 
     @dlt.resource(write_disposition="append", primary_key="id", parallelized=True)
     def flows(
-        updated=dlt.sources.incremental("updated", start_date_obj.isoformat()),
+        updated=dlt.sources.incremental(
+            "updated",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
     ) -> Iterable[TDataItem]:
         intervals = split_date_range(
             pendulum.parse(updated.start_value), pendulum.now()
@@ -146,7 +206,12 @@ def klaviyo_source(api_key: str, start_date: TAnyDateTime) -> Iterable[DltResour
 
     @dlt.resource(write_disposition="append", primary_key="id", parallelized=True)
     def templates(
-        updated=dlt.sources.incremental("updated", start_date_obj.isoformat()),
+        updated=dlt.sources.incremental(
+            "updated",
+            start_date_obj.isoformat(),
+            range_end="closed",
+            range_start="closed",
+        ),
     ) -> Iterable[TDataItem]:
         intervals = split_date_range(
             pendulum.parse(updated.start_value), pendulum.now()

{ingestr-0.12.5 → ingestr-0.12.6}/ingestr/src/shopify/__init__.py

@@ -158,6 +158,8 @@ def shopify_source(
             initial_value=start_date_obj,
             end_value=end_date_obj,
             allow_external_schedulers=True,
+            range_end="closed",
+            range_start="closed",
         ),
         created_at_min: pendulum.DateTime = created_at_min_obj,
         items_per_page: int = items_per_page,
@@ -606,6 +608,8 @@ def shopify_source(
             initial_value=start_date_obj,
             end_value=end_date_obj,
             allow_external_schedulers=True,
+            range_end="closed",
+            range_start="closed",
         ),
         created_at_min: pendulum.DateTime = created_at_min_obj,
         items_per_page: int = items_per_page,
@@ -640,6 +644,8 @@ def shopify_source(
             initial_value=start_date_obj,
             end_value=end_date_obj,
             allow_external_schedulers=True,
+            range_end="closed",
+            range_start="closed",
         ),
         created_at_min: pendulum.DateTime = created_at_min_obj,
         items_per_page: int = items_per_page,
@@ -671,6 +677,8 @@ def shopify_source(
             "created_at",
             initial_value=start_date_obj,
             end_value=end_date_obj,
+            range_end="closed",
+            range_start="closed",
         ),
         items_per_page: int = items_per_page,
     ) -> Iterable[TDataItem]:
@@ -689,6 +697,8 @@ def shopify_source(
             "updated_at",
             initial_value=start_date_obj,
             end_value=end_date_obj,
+            range_end="closed",
+            range_start="closed",
         ),
         items_per_page: int = items_per_page,
     ) -> Iterable[TDataItem]:
@@ -730,6 +740,8 @@ def shopify_source(
             initial_value=start_date_obj,
             end_value=end_date_obj,
             allow_external_schedulers=True,
+            range_end="closed",
+            range_start="closed",
         ),
         items_per_page: int = items_per_page,
     ) -> Iterable[TDataItem]:
@@ -1807,6 +1819,8 @@ query discountNodes($after: String, $query: String, $first: Int)  {
             "updatedAt",
             initial_value=start_date_obj,
             end_value=end_date_obj,
+            range_end="closed",
+            range_start="closed",
         ),
         items_per_page: int = items_per_page,
     ) -> Iterable[TDataItem]:

{ingestr-0.12.5 → ingestr-0.12.6}/ingestr/src/slack/__init__.py

@@ -175,6 +175,8 @@ def slack_source(
             initial_value=start_dt,
             end_value=end_dt,
             allow_external_schedulers=True,
+            range_end="closed",
+            range_start="closed",
         ),
     ) -> Iterable[TDataItem]:
         """
@@ -198,6 +200,8 @@ def slack_source(
             initial_value=start_dt,
             end_value=end_dt,
             allow_external_schedulers=True,
+            range_end="closed",
+            range_start="closed",
         ),
     ) -> Iterable[TDataItem]:
         """Yield all messages for a given channel as a DLT resource. Keep blocks column without normalization.