territories-dashboard-lib 0.1.39b2__py3-none-any.whl → 1.1.1.dev10__py3-none-any.whl

territories_dashboard_lib/indicators_lib/query/commons.py

@@ -1,9 +1,55 @@
-from ..enums import (
-    FRANCE_DB_VALUES,
-    MeshLevel,
-)
+from territories_dashboard_lib.indicators_lib.payloads import Territory
+
+from ..enums import FRANCE_DB_VALUES, MeshLevel
 from ..models import AggregationFunctions, Indicator
-from .utils import get_breakdown_dimension, run_custom_query
+from .utils import format_sql_codes, get_breakdown_dimension, run_custom_query
+
+
+def get_mesh_column_name(mesh: MeshLevel):
+    return f"code_{mesh}"
+
+
+def get_sub_territories(
+    *,
+    submesh: MeshLevel,
+    territory: Territory = None,
+    with_center=False,
+    codes: list[str] = None,
+):
+    """
+    Récupère une liste des territoires appartenant à un territoire parent.
+    params:
+        submesh: maille des sous-territoires à récupérer
+        territory: territoire parent des sous-territoires
+        with_center: si True, ajoute le centre de chaque territoire
+        codes: codes des sous-territoires à récupérer, peut-être utilisé en absence du territoire parent
+    """
+    parent_mesh_join = ""
+    condition = ""
+    if territory is None and codes:
+        condition = f"WHERE arbo.code IN {format_sql_codes(codes)}"
+    elif territory and submesh == territory.mesh:
+        condition = f"WHERE arbo.code IN {territory.sql_codes}"
+    elif territory:
+        parent_mesh_join = f"JOIN arbo_{territory.mesh}_{submesh} AS j ON j.{submesh}=arbo.code AND j.{territory.mesh} IN {territory.sql_codes}"
+
+    center = ""
+    contours_join = ""
+    if with_center:
+        center = ", ST_ASGEOJSON(ST_CENTROID(contours.geometry)) as center"
+        contours_join = f"JOIN contours_simplified_{submesh} as contours ON arbo.code = contours.code"
+
+    query = f"""
+        SELECT DISTINCT
+            arbo.code,
+            arbo.name
+            {center}
+        FROM arbo_{submesh} AS arbo
+        {parent_mesh_join}
+        {contours_join}
+        {condition}
+    """
+    return run_custom_query(query)
 
 
 def get_last_year(indicator, mesh):
@@ -18,9 +64,9 @@ def generate_aggregate_query(indicator, territory, submesh, filters, slicer):
     aggregat AS
     (
         SELECT
-        {calculate_aggregate_values(indicator)}, {slicer}
+        {calculate_aggregate_values(indicator)}, indic.{slicer}
         FROM
-        {get_table_data_for_geography(indicator, territory, submesh)}
+        {get_values_for_indicator_territory_submesh(indicator=indicator, territory=territory, submesh=submesh)}
         {add_optional_filters(indicator, filters)}
         AND annee = (SELECT * FROM annee_max)
         GROUP BY (indic.{slicer})
@@ -30,9 +76,8 @@ def generate_aggregate_query(indicator, territory, submesh, filters, slicer):
 
 
 def generate_aggregate_query_for_location(indicator, territory, submesh, filters):
-    return generate_aggregate_query(
-        indicator, territory, submesh, filters, f"code_{submesh}"
-    )
+    mesh_col = get_mesh_column_name(submesh)
+    return generate_aggregate_query(indicator, territory, submesh, filters, mesh_col)
 
 
 def add_optional_filters(indicator: Indicator, filters):
@@ -47,75 +92,48 @@ def add_optional_filters(indicator: Indicator, filters):
     return condition
 
 
-def get_place_names_join(include_place_name, mapped_submesh, submesh, flows=False):
-    join_clause = ""
-    if include_place_name:
-        if flows:
-            join_clause = f"""
-            JOIN
-                (SELECT distinct("NOM_{mapped_submesh}") as territory_1, "{mapped_submesh}" as territory_1_id, "{mapped_submesh}" from arborescence_geo) arbo1
-                on arbo1."{mapped_submesh}" = indic.code_{submesh.lower()}_1
-            JOIN
-                (SELECT distinct("NOM_{mapped_submesh}") as territory_2, "{mapped_submesh}" as territory_2_id, "{mapped_submesh}" from arborescence_geo) arbo2
-                on arbo2."{mapped_submesh}" = indic.code_{submesh.lower()}_2
-            """
-        else:
-            join_clause = f"""
-                JOIN
-                (SELECT distinct("NOM_{mapped_submesh}") as lieu, "{mapped_submesh}" as territoryid, "{mapped_submesh}" from arborescence_geo) arbo
-                on arbo."{mapped_submesh}" = indic.code_{submesh.lower()}
-            """
-    return join_clause
-
-
-def get_table_data_for_geography(
-    indicator,
-    territory,
-    submesh=MeshLevel.Region,
-    include_place_names=None,
-    flows=False,
+def get_values_for_indicator_territory_submesh(
+    *,
+    indicator: Indicator,
+    territory: Territory,
+    submesh: MeshLevel,
+    flows: bool = False,
 ):
-    mapped_mesh = "DEPCOM" if territory.mesh == "com" else territory.mesh.upper()
-    mapped_submesh = "DEPCOM" if submesh == "com" else submesh.upper()
-
-    geo_id_values = "', '".join([id.strip() for id in territory.id.split(",")])
-
-    table_prefix = (
-        indicator.flows_db_table_prefix if flows else indicator.db_table_prefix
-    )
-
-    arbo_sub_query = f"""
-    SELECT DISTINCT("{mapped_submesh}")
-    FROM arborescence_geo arbo
-    WHERE arbo."{mapped_mesh}" in('{geo_id_values}')
-    """
-
-    where = (
-        f"""
-        WHERE (indic.code_{submesh.lower()}_1 in ({arbo_sub_query})
-        OR indic.code_{submesh.lower()}_2 in ({arbo_sub_query}))
+    mesh_col = get_mesh_column_name(submesh)
+    if submesh == territory.mesh:
+        arbo_join = f"""
+        JOIN arbo_{submesh} arbo ON arbo.code = indic.{mesh_col} AND arbo.code IN {territory.sql_codes}
         """
-        if flows
-        else f"""
-        WHERE indic.code_{submesh.lower()} in ({arbo_sub_query})
+    else:
+        arbo_join = f"""
+        JOIN arbo_{territory.mesh}_{submesh} j 
+            ON j.{submesh} = indic.{mesh_col} AND j.{territory.mesh} IN {territory.sql_codes}
+        JOIN arbo_{submesh} arbo
+            ON arbo.code = indic.{mesh_col}
+        """
+    if flows:
+        arbo_join = f"""
+        JOIN arbo_{territory.mesh}_{submesh} j1 
+            ON j1.{submesh} = indic.{mesh_col}_1 AND j1.{territory.mesh} IN {territory.sql_codes}
+        JOIN arbo_{submesh} arbo1
+            ON arbo1.code = indic.{mesh_col}_1
+
+        JOIN arbo_{territory.mesh}_{submesh} j2
+            ON j2.{submesh} = indic.{mesh_col}_2 AND j2.{territory.mesh} IN {territory.sql_codes}
+        JOIN arbo_{submesh} arbo2
+            ON arbo2.code = indic.{mesh_col}_2
         """
-    )
 
     return f"""
-    "{table_prefix}_{submesh.lower()}" indic
-    {get_place_names_join(include_place_names, mapped_submesh, submesh, flows)}
-    {where}
+    "{indicator.table_name(submesh, flows=flows)}" indic
+    {arbo_join}
     """
 
 
 def calculate_aggregate_values(indicator, with_alternative=True):
-    # TODO coverage tester avec un indicateur de cette sorte
-    # Bastien doit exporter des nouvelles données pour faire le test
     if not indicator.is_composite:
         return "SUM(valeur) as valeur"
 
-    # TODO coverage tester avec un indicateur de cette sorte
-    # Bastien doit exporter des nouvelles données pour faire le test
     if indicator.aggregation_function == AggregationFunctions.DISCRETE_COMPONENT_2:
         sql = f"SUM(composante_1) / COALESCE(NULLIF(SUM(composante_2), 0), 1) * {indicator.aggregation_constant} as valeur"
         if with_alternative:
@@ -133,45 +151,14 @@ def calculate_aggregate_values(indicator, with_alternative=True):
     return sql
 
 
-def get_territories_ids(main_territory_codes, territory_mesh, submesh):
-    mapped_territory_mesh = (
-        "DEPCOM" if territory_mesh == "com" else territory_mesh.upper()
-    )
-    mapped_submesh = "DEPCOM" if submesh == "com" else submesh.upper()
-
-    query = f"""
-        SELECT DISTINCT "{mapped_submesh}" as code
-        FROM arborescence_geo
-        WHERE "{mapped_territory_mesh}" IN ('{"', '".join(main_territory_codes)}')
-        """
-
-    territories_ids = [r["code"] for r in run_custom_query(query)]
-    return territories_ids
-
-
-def get_sub_territories(territory, submesh):
-    territory_codes = territory.id.split(",")
-    mapped_territory_mesh = (
-        "DEPCOM" if territory.mesh == "com" else territory.mesh.upper()
-    )
-    mapped_submesh = "DEPCOM" if submesh == "com" else submesh.upper()
-
-    query = f"""
-        SELECT DISTINCT "{mapped_submesh}" as code, "NOM_{mapped_submesh}" as name
-        FROM arborescence_geo
-        WHERE "{mapped_territory_mesh}" IN ('{"', '".join(territory_codes)}')
-        """
-
-    return run_custom_query(query)
-
-
 def get_where_territory(territory):
     territory_id = (
         FRANCE_DB_VALUES[territory.id]
-        if territory.mesh == MeshLevel.National
+        if territory.mesh == MeshLevel.fr
         else territory.id
     )
-    return f""" "code_{territory.mesh}" = '{territory_id}' """
+    mesh_col = get_mesh_column_name(territory.mesh)
+    return f""" "{mesh_col}" = '{territory_id}' """
 
 
 def get_values_for_territory(indicator, territory, filters=None):
@@ -189,7 +176,6 @@ def get_values_for_territory(indicator, territory, filters=None):
 
 
 def get_territory_name(territory):
-    territory_mesh = "DEPCOM" if territory.mesh == "com" else territory.mesh.upper()
-    query = f"""SELECT "NOM_{territory_mesh}" as nom FROM arborescence_geo WHERE "{territory_mesh}" = '{territory.id}' LIMIT 1;"""
+    query = f"""SELECT name FROM arbo_{territory.mesh} WHERE code IN {territory.sql_codes} LIMIT 1;"""
     results = run_custom_query(query)
-    return results[0]["nom"] if results else ""
+    return results[0]["name"] if results else ""

territories_dashboard_lib/indicators_lib/query/comparison.py

@@ -1,6 +1,10 @@
 from collections import defaultdict
 
-from .commons import add_optional_filters, calculate_aggregate_values
+from .commons import (
+    add_optional_filters,
+    calculate_aggregate_values,
+    get_mesh_column_name,
+)
 from .utils import run_custom_query
 
 
@@ -13,12 +17,13 @@ def get_comparison_values_and_buckets(
     filters = add_optional_filters(indicator, filters)
     value = calculate_aggregate_values(indicator, with_alternative=False)
     number_of_buckets = min(10, len(territories) + len(cmp_territories))
+    mesh_col = get_mesh_column_name(submesh)
     sub_query = f"""
     SELECT
         {value},
-        code_{submesh} as geo_code
+        {mesh_col} as geo_code
     FROM {table_name} indic
-    WHERE code_{submesh} IN {territories_sql}
+    WHERE {mesh_col} IN {territories_sql}
     AND annee = (
         SELECT MAX(annee)
         FROM {table_name}

territories_dashboard_lib/indicators_lib/query/details.py

@@ -5,7 +5,7 @@ from .commons import (
     add_optional_filters,
     calculate_aggregate_values,
     get_last_year,
-    get_place_names_join,
+    get_mesh_column_name,
     get_where_territory,
 )
 from .utils import get_breakdown_dimension, run_custom_query
@@ -23,7 +23,6 @@ def get_proportions_chart(indicator, territory, filters):
     {add_optional_filters(indicator, filters)}
     GROUP BY "{breakdown_dimension.db_name}"
 """
-
     data = run_custom_query(query)
     filters_color = {f.db_name: f.color for f in breakdown_dimension.filters.all()}
     data_dict = {
@@ -47,19 +46,15 @@ def get_values_for_submesh_territories(
     indicator: Indicator, submesh: MeshLevel, territory: Territory, filters
 ):
     submesh = submesh.lower()
-    territory_mesh = "DEPCOM" if territory.mesh == "com" else territory.mesh.upper()
-    mapped_submesh = "DEPCOM" if submesh == "com" else submesh.upper()
-
-    geo_ids = "', '".join([id.strip() for id in territory.id.split(",")])
+    mesh_col = get_mesh_column_name(submesh)
+    parent_territory_join = f"JOIN arbo_{territory.mesh}_{submesh} j ON j.{submesh} = arbo.code AND j.{territory.mesh} IN {territory.sql_codes}"
+    if submesh == territory.mesh:
+        parent_territory_join = f"AND arbo.code IN {territory.sql_codes}"
     query = f"""
-    SELECT {calculate_aggregate_values(indicator)}, code_{submesh} as geocode, arbo.lieu as geoname
+    SELECT {calculate_aggregate_values(indicator)}, indic.{mesh_col} as geocode, arbo.name as geoname
     FROM "{indicator.db_table_prefix}_{submesh}" indic
-    {get_place_names_join(True, mapped_submesh, submesh)}
-    WHERE code_{submesh} IN (
-        SELECT DISTINCT("{mapped_submesh}")
-        FROM arborescence_geo arbo
-        WHERE arbo."{territory_mesh}" IN ('{geo_ids}')
-    )
+    JOIN arbo_{submesh} arbo ON arbo.code = indic.{mesh_col}
+    {parent_territory_join}
     {add_optional_filters(indicator, filters)}
     AND annee = (
         SELECT MAX(annee)

territories_dashboard_lib/indicators_lib/query/histogram.py

@@ -77,7 +77,6 @@ def get_indicator_histogram_data(indicator, territory, submesh, filters):
     territory_values = get_values_for_submesh_territories(
         indicator, submesh, territory, filters
     )
-    print("!!", territory_values)
     datasets_histogram_bar_chart, deciles = get_territories_histogram_data(
         territory_values, indicator.unite
     )

territories_dashboard_lib/indicators_lib/query/indicator_card.py

@@ -1,4 +1,7 @@
-from .commons import generate_aggregate_query_for_location
+from .commons import (
+    generate_aggregate_query_for_location,
+    get_mesh_column_name,
+)
 from .utils import run_custom_query
 
 
@@ -29,15 +32,16 @@ def get_min_max_statistics(indicator, submesh):
 
 
 def get_values_for(extremum, submesh, is_alternative=False):
+    mesh_col = get_mesh_column_name(submesh)
     return f"""
     {extremum}_value,
     (
-        SELECT code_{submesh} AS code_{extremum}
+        SELECT {mesh_col} AS code_{extremum}
         FROM aggregat
         WHERE valeur{"_alternative" if is_alternative else ""} = (SELECT * FROM {extremum}_value) LIMIT 1
     ) AS code_{extremum},
     (
-        SELECT count(code_{submesh}) AS count_{extremum}
+        SELECT count({mesh_col}) AS count_{extremum}
         FROM aggregat
         WHERE valeur{"_alternative" if is_alternative else ""} = (SELECT * FROM {extremum}_value)
     ) AS count_{extremum},
@@ -73,12 +77,13 @@ def get_names_from_codes(dict_result, submesh):
     for key, value in dict_result.items():
         if key.startswith("code_"):
             codes_to_fetch.append(value)
-    mapped_submesh = "DEPCOM" if submesh == "com" else submesh.upper()
+    codes = ", ".join(f"'{code.strip()}'" for code in codes_to_fetch)
     query = f"""
-    SELECT DISTINCT "{mapped_submesh}" as code, "NOM_{mapped_submesh}" as name
-    FROM arborescence_geo
-    WHERE "{mapped_submesh}" IN ({", ".join(["'" + c + "'" for c in codes_to_fetch])});
+    SELECT code, name
+    FROM arbo_{submesh}
+    WHERE code IN ({codes});
     """
+
     rows = run_custom_query(query)
     names_dict = {}
     for row in rows:

territories_dashboard_lib/indicators_lib/query/top_10.py

@@ -4,7 +4,8 @@ from .commons import (
     add_optional_filters,
     calculate_aggregate_values,
     get_last_year,
-    get_table_data_for_geography,
+    get_mesh_column_name,
+    get_values_for_indicator_territory_submesh,
 )
 from .utils import get_breakdown_dimension, run_custom_query
 
@@ -14,29 +15,30 @@ def breakdown_territories(indicator, submesh, year, filters, top_10_territories)
         [f"'{territory['code_geo']}'" for territory in top_10_territories]
     )
     breakdown_dimension = get_breakdown_dimension(indicator).db_name
-
+    mesh_col = get_mesh_column_name(submesh)
     query = f"""
         SELECT {calculate_aggregate_values(indicator, with_alternative=False)},
             "{breakdown_dimension}" as dimension,
-            "code_{submesh}" as code_geo
+            "{mesh_col}" as code_geo
         FROM "{indicator.db_table_prefix}_{submesh}" as indic
-        WHERE code_{submesh} in ({territories_code}) AND annee = {year}
+        WHERE {mesh_col} in ({territories_code}) AND annee = {year}
         {add_optional_filters(indicator, filters)}
-        GROUP BY "code_{submesh}", "{breakdown_dimension}"
+        GROUP BY "{mesh_col}", "{breakdown_dimension}"
     """
     results = run_custom_query(query)
     return results
 
 
 def get_top_10_territories(indicator, territory, submesh, year, filters):
+    mesh_col = get_mesh_column_name(submesh)
     query = f"""
     SELECT {calculate_aggregate_values(indicator, with_alternative=False)},
-    indic.code_{submesh} as code_geo, arbo.lieu as lieu
-    FROM {get_table_data_for_geography(indicator, territory, submesh, include_place_names=True)}
+    indic.{mesh_col} as code_geo, arbo.name
+    FROM {get_values_for_indicator_territory_submesh(indicator=indicator, territory=territory, submesh=submesh)}
     {add_optional_filters(indicator, filters)}
     AND annee = {year}
     AND valeur IS NOT NULL
-    GROUP BY code_{submesh}, arbo.lieu
+    GROUP BY indic.{mesh_col}, arbo.name
     ORDER BY valeur DESC
     LIMIT 10
     """
@@ -78,8 +80,6 @@ def get_indicator_top_10_data(indicator, territory, submesh, filters):
             }
             for f in breakdown_filters
         ]
-    # TODO coverage faire un test avec un indicateur qui n'a pas de dimensions
-    # necessite un export de données de Bastien
     else:
         datasets_top_bar_chart = [
             {
@@ -88,12 +88,12 @@ def get_indicator_top_10_data(indicator, territory, submesh, filters):
             }
         ]
 
-    labels_top_bar_chart = [territory["lieu"] for territory in territories]
+    labels_top_bar_chart = [territory["name"] for territory in territories]
 
     csv_data = []
     for territory in territories:
         csv_row = {}
-        csv_row["Territoire"] = territory["lieu"]
+        csv_row["Territoire"] = territory["name"]
         csv_row["Code Géographique"] = territory["code_geo"]
         csv_row[f"Valeur {indicator.unite}"] = territory["valeur"]
         if breakdown_dimension:

territories_dashboard_lib/indicators_lib/query/utils.py

@@ -2,6 +2,15 @@ from django.conf import settings
 from django.db import connections
 
 
+def format_sql_codes(codes: list[str]):
+    """
+    Format list of IDs for SQL IN statement.
+    ["1","2","3"] => ('1', '2', '3')
+    """
+    codes = ", ".join(f"'{code.strip()}'" for code in codes)
+    return f"({codes})"
+
+
 def get_breakdown_dimension(indicator):
     breakdown_dimension = (
         indicator.dimensions.filter(is_breakdown=True).first()

territories_dashboard_lib/indicators_lib/table.py

@@ -1,5 +1,8 @@
 from .payloads import IndicatorTablePayload
-from .query.commons import add_optional_filters, get_table_data_for_geography
+from .query.commons import (
+    add_optional_filters,
+    get_values_for_indicator_territory_submesh,
+)
 from .query.utils import run_custom_query
 
 
@@ -22,10 +25,10 @@ def _get_query(
         )
     )
     territory_search_where = (
-        "OR (unaccent(LOWER(territory_1)) LIKE unaccent(LOWER(CONCAT('%%', %(search)s, '%%')))) OR (unaccent(LOWER(territory_2)) LIKE unaccent(LOWER(CONCAT('%%', %(search)s, '%%'))))"
+        "OR (unaccent(LOWER(arbo1.name)) LIKE unaccent(LOWER(CONCAT('%%', %(search)s, '%%')))) OR (unaccent(LOWER(arbo2.name)) LIKE unaccent(LOWER(CONCAT('%%', %(search)s, '%%'))))"
         if props.flows
         else (
-            "OR (unaccent(LOWER(lieu)) LIKE unaccent(LOWER(CONCAT('%%', %(search)s, '%%'))))"
+            "OR (unaccent(LOWER(name)) LIKE unaccent(LOWER(CONCAT('%%', %(search)s, '%%'))))"
         )
     )
     where = (
@@ -48,7 +51,7 @@ def _get_query(
         SELECT
             {", ".join(columns)}
         FROM
-            {get_table_data_for_geography(indicator, props.territory, props.submesh, True, props.flows)}
+            {get_values_for_indicator_territory_submesh(indicator=indicator, territory=props.territory, submesh=props.submesh, flows=props.flows)}
             {regular_dimensions_filters}
         {where} {where_year}
         {order_clause}
@@ -60,18 +63,18 @@ def get_values_columns(indicator, props, *, for_export=False):
     if props.flows:
         columns = [
             "annee",
-            "territory_1",
-            "territory_2",
+            "arbo1.name as territory_1",
+            "arbo2.name as territory_2",
             f"{indicator.flows_dimension} AS dimension"
             if indicator.flows_dimension
             else None,
             "CAST(valeur AS int) as valeur",
         ]
         if for_export:
-            columns += ["territory_1_id", "territory_2_id"]
+            columns += ["arbo1.code as territory_1_id", "arbo2.code as territory_2_id"]
     else:
         columns = (
-            ["annee", "lieu"]
+            ["annee", "name"]
             + [f"{dimension.db_name}" for dimension in indicator.dimensions.all()]
             + [
                 f"valeur * {indicator.aggregation_constant if indicator.unite == '%' else 1} AS valeur",
@@ -81,17 +84,17 @@ def get_values_columns(indicator, props, *, for_export=False):
             ]
         )
         if for_export:
-            columns += ["territoryid"]
+            columns += ["code"]
     return columns
 
 
 def get_values_orders(indicator, props: IndicatorTablePayload):
     territory_order = None
-    if props.column_order not in ["lieu", "territory_1", "territory_2"]:
+    if props.column_order not in ["name", "arbo1.name", "arbo2.name"]:
         territory_order = (
-            "LOWER(territory_1) ASC, LOWER(territory_2) ASC"
+            "LOWER(arbo1.name) ASC, LOWER(arbo2.name) ASC"
             if props.flows
-            else "LOWER(lieu) ASC"
+            else "LOWER(name) ASC"
         )
     year_order = "annee DESC" if props.column_order != "annee" else None
     dimension_order = None