OpenFisca-Core-test 46.0.0__py3-none-any.whl

openfisca_web_api/app.py

@@ -0,0 +1,226 @@
+import logging
+import os
+import traceback
+
+from openfisca_core.errors import PeriodMismatchError, SituationParsingError
+from openfisca_web_api import handlers
+from openfisca_web_api.errors import handle_import_error
+from openfisca_web_api.loader import build_data
+
+try:
+    import werkzeug.exceptions
+    from flask import Flask, abort, jsonify, make_response, redirect, request
+    from flask_cors import CORS
+    from werkzeug.middleware.proxy_fix import ProxyFix
+except ImportError as error:
+    handle_import_error(error)
+
+log = logging.getLogger("gunicorn.error")
+
+
+def init_tracker(url, idsite, tracker_token):
+    try:
+        from openfisca_tracker.piwik import PiwikTracker
+
+        tracker = PiwikTracker(url, idsite, tracker_token)
+
+        info = os.linesep.join(
+            [
+                "You chose to activate the `tracker` module. ",
+                "Tracking data will be sent to: " + url,
+                "For more information, see <https://github.com/openfisca/openfisca-core#tracker-configuration>.",
+            ],
+        )
+        log.info(info)
+        return tracker
+
+    except ImportError:
+        message = os.linesep.join(
+            [
+                traceback.format_exc(),
+                "You chose to activate the `tracker` module, but it is not installed.",
+                "For more information, see <https://github.com/openfisca/openfisca-core#tracker-installation>.",
+            ],
+        )
+        log.warning(message)
+
+
+def create_app(
+    tax_benefit_system,
+    tracker_url=None,
+    tracker_idsite=None,
+    tracker_token=None,
+    welcome_message=None,
+):
+    if not tracker_url or not tracker_idsite:
+        tracker = None
+    else:
+        tracker = init_tracker(tracker_url, tracker_idsite, tracker_token)
+
+    app = Flask(__name__)
+    # Fix request.remote_addr to get the real client IP address
+    app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1, x_host=1)
+    CORS(app, origins="*")
+
+    app.config["JSON_AS_ASCII"] = False  # When False, lets jsonify encode to utf-8
+    app.url_map.strict_slashes = False  # Accept url like /parameters/
+    app.url_map.merge_slashes = False  # Do not eliminate // in paths
+    app.config["JSON_SORT_KEYS"] = False  # Don't sort JSON keys in the Web API
+
+    data = build_data(tax_benefit_system)
+
+    DEFAULT_WELCOME_MESSAGE = "This is the root of an OpenFisca Web API. To learn how to use it, check the general documentation (https://openfisca.org/doc/) and the OpenAPI specification of this instance ({}spec)."
+
+    @app.before_request
+    def before_request():
+        if request.path != "/" and request.path.endswith("/"):
+            return redirect(request.path[:-1])
+        return None
+
+    @app.route("/")
+    def get_root():
+        return (
+            jsonify(
+                {
+                    "welcome": welcome_message
+                    or DEFAULT_WELCOME_MESSAGE.format(request.host_url),
+                },
+            ),
+            300,
+        )
+
+    @app.route("/parameters")
+    def get_parameters():
+        parameters = {
+            parameter["id"]: {
+                "description": parameter["description"],
+                "href": f"{request.host_url}parameter/{name}",
+            }
+            for name, parameter in data["parameters"].items()
+            if parameter.get("subparams")
+            is None  # For now and for backward compat, don't show nodes in overview
+        }
+
+        return jsonify(parameters)
+
+    @app.route("/parameter/<path:parameter_id>")
+    def get_parameter(parameter_id):
+        parameter = data["parameters"].get(parameter_id)
+        if parameter is None:
+            # Try legacy route
+            parameter_new_id = parameter_id.replace(".", "/")
+            parameter = data["parameters"].get(parameter_new_id)
+        if parameter is None:
+            raise abort(404)
+        return jsonify(parameter)
+
+    @app.route("/variables")
+    def get_variables():
+        variables = {
+            name: {
+                "description": variable["description"],
+                "href": f"{request.host_url}variable/{name}",
+            }
+            for name, variable in data["variables"].items()
+        }
+        return jsonify(variables)
+
+    @app.route("/variable/<id>")
+    def get_variable(id):
+        variable = data["variables"].get(id)
+        if variable is None:
+            raise abort(404)
+        return jsonify(variable)
+
+    @app.route("/entities")
+    def get_entities():
+        return jsonify(data["entities"])
+
+    @app.route("/spec")
+    def get_spec():
+        scheme = request.environ["wsgi.url_scheme"]
+        host = request.host
+        url = f"{scheme}://{host}"
+
+        return jsonify(
+            {
+                **data["openAPI_spec"],
+                "servers": [{"url": url}],
+            },
+        )
+
+    def handle_invalid_json(error) -> None:
+        json_response = jsonify(
+            {
+                "error": f"Invalid JSON: {error.args[0]}",
+            },
+        )
+
+        abort(make_response(json_response, 400))
+
+    @app.route("/calculate", methods=["POST"])
+    def calculate():
+        tax_benefit_system = data["tax_benefit_system"]
+        request.on_json_loading_failed = handle_invalid_json
+        input_data = request.get_json()
+        try:
+            result = handlers.calculate(tax_benefit_system, input_data)
+        except (SituationParsingError, PeriodMismatchError) as e:
+            abort(make_response(jsonify(e.error), e.code or 400))
+        except (UnicodeEncodeError, UnicodeDecodeError) as e:
+            abort(
+                make_response(
+                    jsonify({"error": "'" + e[1] + "' is not a valid ASCII value."}),
+                    400,
+                ),
+            )
+        return jsonify(result)
+
+    @app.route("/trace", methods=["POST"])
+    def trace():
+        tax_benefit_system = data["tax_benefit_system"]
+        request.on_json_loading_failed = handle_invalid_json
+        input_data = request.get_json()
+        try:
+            result = handlers.trace(tax_benefit_system, input_data)
+        except (SituationParsingError, PeriodMismatchError) as e:
+            abort(make_response(jsonify(e.error), e.code or 400))
+        return jsonify(result)
+
+    @app.after_request
+    def apply_headers(response):
+        response.headers.extend(
+            {
+                "Country-Package": data["country_package_metadata"]["name"],
+                "Country-Package-Version": data["country_package_metadata"]["version"],
+            },
+        )
+        return response
+
+    @app.after_request
+    def track_requests(response):
+        if tracker:
+            if request.headers.get("dnt"):
+                source_ip = ""
+            elif request.headers.get("X-Forwarded-For"):
+                source_ip = request.headers["X-Forwarded-For"].split(", ")[0]
+            else:
+                source_ip = request.remote_addr
+
+            api_version = "{}-{}".format(
+                data["country_package_metadata"]["name"],
+                data["country_package_metadata"]["version"],
+            )
+
+            tracker.track(request.url, source_ip, api_version, request.path)
+        return response
+
+    @app.errorhandler(500)
+    def internal_server_error(e: werkzeug.exceptions.InternalServerError):
+        message = getattr(e, "original_exception", e.description)
+        response = jsonify({"error": "Internal server error: " + str(message)})
+        response.status_code = 500
+
+        return response
+
+    return app

openfisca_web_api/errors.py

@@ -0,0 +1,11 @@
+import logging
+from typing import NoReturn
+
+log = logging.getLogger("gunicorn.error")
+
+
+def handle_import_error(error) -> NoReturn:
+    msg = f"OpenFisca is missing some dependencies to run the Web API: '{error}'. To install them, run `pip install openfisca_core[web-api]`."
+    raise ImportError(
+        msg,
+    )

openfisca_web_api/handlers.py

@@ -0,0 +1,90 @@
+import dpath
+
+from openfisca_core.indexed_enums import Enum
+from openfisca_core.simulations import SimulationBuilder
+
+
+def calculate(tax_benefit_system, input_data: dict) -> dict:
+    """Returns the input_data where the None values are replaced by the calculated values."""
+    simulation = SimulationBuilder().build_from_entities(tax_benefit_system, input_data)
+    requested_computations = dpath.search(
+        input_data,
+        "*/*/*/*",
+        afilter=lambda t: t is None,
+        yielded=True,
+    )
+    computation_results: dict = {}
+    for computation in requested_computations:
+        path = computation[
+            0
+        ]  # format: entity_plural/entity_instance_id/openfisca_variable_name/period
+        entity_plural, entity_id, variable_name, period = path.split("/")
+        variable = tax_benefit_system.get_variable(variable_name)
+        result = simulation.calculate(variable_name, period)
+        population = simulation.get_population(entity_plural)
+        entity_index = population.get_index(entity_id)
+
+        if variable.value_type == Enum:
+            entity_result = result.decode()[entity_index].name
+        elif variable.value_type == float:
+            entity_result = float(
+                str(result[entity_index]),
+            )  # To turn the float32 into a regular float without adding confusing extra decimals. There must be a better way.
+        elif variable.value_type == str:
+            entity_result = str(result[entity_index])
+        else:
+            entity_result = result.tolist()[entity_index]
+        # Don't use dpath.new, because there is a problem with dpath>=2.0
+        # when we have a key that is numeric, like the year.
+        # See https://github.com/dpath-maintainers/dpath-python/issues/160
+        if computation_results == {}:
+            computation_results = {
+                entity_plural: {entity_id: {variable_name: {period: entity_result}}},
+            }
+        elif entity_plural in computation_results:
+            if entity_id in computation_results[entity_plural]:
+                if variable_name in computation_results[entity_plural][entity_id]:
+                    computation_results[entity_plural][entity_id][variable_name][
+                        period
+                    ] = entity_result
+                else:
+                    computation_results[entity_plural][entity_id][variable_name] = {
+                        period: entity_result,
+                    }
+            else:
+                computation_results[entity_plural][entity_id] = {
+                    variable_name: {period: entity_result},
+                }
+        else:
+            computation_results[entity_plural] = {
+                entity_id: {variable_name: {period: entity_result}},
+            }
+    dpath.merge(input_data, computation_results)
+
+    return input_data
+
+
+def trace(tax_benefit_system, input_data):
+    simulation = SimulationBuilder().build_from_entities(tax_benefit_system, input_data)
+    simulation.trace = True
+
+    requested_calculations = []
+    requested_computations = dpath.search(
+        input_data,
+        "*/*/*/*",
+        afilter=lambda t: t is None,
+        yielded=True,
+    )
+    for computation in requested_computations:
+        path = computation[0]
+        _entity_plural, _entity_id, variable_name, period = path.split("/")
+        requested_calculations.append(f"{variable_name}<{period!s}>")
+        simulation.calculate(variable_name, period)
+
+    trace = simulation.tracer.get_serialized_flat_trace()
+
+    return {
+        "trace": trace,
+        "entitiesDescription": simulation.describe_entities(),
+        "requestedCalculations": requested_calculations,
+    }

openfisca_web_api/loader/__init__.py

@@ -0,0 +1,22 @@
+from openfisca_web_api.loader.entities import build_entities
+from openfisca_web_api.loader.parameters import build_parameters
+from openfisca_web_api.loader.spec import build_openAPI_specification
+from openfisca_web_api.loader.variables import build_variables
+
+
+def build_data(tax_benefit_system):
+    country_package_metadata = tax_benefit_system.get_package_metadata()
+    parameters = build_parameters(tax_benefit_system, country_package_metadata)
+    variables = build_variables(tax_benefit_system, country_package_metadata)
+    entities = build_entities(tax_benefit_system)
+    data = {
+        "tax_benefit_system": tax_benefit_system,
+        "country_package_metadata": country_package_metadata,
+        "openAPI_spec": None,
+        "parameters": parameters,
+        "variables": variables,
+        "entities": entities,
+    }
+    data["openAPI_spec"] = build_openAPI_specification(data)
+
+    return data

openfisca_web_api/loader/entities.py

@@ -0,0 +1,28 @@
+def build_entities(tax_benefit_system):
+    return {entity.key: build_entity(entity) for entity in tax_benefit_system.entities}
+
+
+def build_entity(entity):
+    formatted_doc = entity.doc.strip()
+
+    formatted_entity = {
+        "plural": entity.plural,
+        "description": entity.label,
+        "documentation": formatted_doc,
+    }
+    if not entity.is_person:
+        formatted_entity["roles"] = {
+            role.key: build_role(role) for role in entity.roles
+        }
+    return formatted_entity
+
+
+def build_role(role):
+    formatted_role = {"plural": role.plural, "description": role.doc}
+
+    if role.max:
+        formatted_role["max"] = role.max
+    if role.subroles:
+        formatted_role["max"] = len(role.subroles)
+
+    return formatted_role

openfisca_web_api/loader/parameters.py

@@ -0,0 +1,122 @@
+import functools
+import operator
+
+from openfisca_core.parameters import Parameter, ParameterNode, Scale
+
+
+def build_api_values_history(values_history):
+    api_values_history = {}
+    for value_at_instant in values_history.values_list:
+        api_values_history[value_at_instant.instant_str] = value_at_instant.value
+
+    return api_values_history
+
+
+def get_value(date, values):
+    candidates = sorted(
+        [
+            (start_date, value)
+            for start_date, value in values.items()
+            if start_date
+            <= date  # dates are lexicographically ordered and can be sorted
+        ],
+        reverse=True,
+    )
+
+    if candidates:
+        return candidates[0][1]
+    return None
+
+
+def build_api_scale(scale, value_key_name):
+    # preprocess brackets for a scale with 'rates' or 'amounts'
+    brackets = [
+        {
+            "thresholds": build_api_values_history(bracket.threshold),
+            "values": build_api_values_history(getattr(bracket, value_key_name)),
+        }
+        for bracket in scale.brackets
+    ]
+
+    dates = set(
+        functools.reduce(
+            operator.iadd,
+            [
+                list(bracket["thresholds"].keys()) + list(bracket["values"].keys())
+                for bracket in brackets
+            ],
+            [],
+        ),
+    )  # flatten the dates and remove duplicates
+
+    # We iterate on all dates as we need to build the whole scale for each of them
+    api_scale = {}
+    for date in dates:
+        for bracket in brackets:
+            threshold_value = get_value(date, bracket["thresholds"])
+            if threshold_value is not None:
+                rate_or_amount_value = get_value(date, bracket["values"])
+                api_scale[date] = api_scale.get(date) or {}
+                api_scale[date][threshold_value] = rate_or_amount_value
+
+    # Handle stopped parameters: a parameter is stopped if its first bracket is stopped
+    latest_date_first_threshold = max(brackets[0]["thresholds"].keys())
+    latest_value_first_threshold = brackets[0]["thresholds"][
+        latest_date_first_threshold
+    ]
+
+    if latest_value_first_threshold is None:
+        api_scale[latest_date_first_threshold] = None
+
+    return api_scale
+
+
+def build_source_url(absolute_file_path, country_package_metadata):
+    relative_path = absolute_file_path.replace(country_package_metadata["location"], "")
+    return "{}/blob/{}{}".format(
+        country_package_metadata["repository_url"],
+        country_package_metadata["version"],
+        relative_path,
+    )
+
+
+def build_api_parameter(parameter, country_package_metadata):
+    api_parameter = {
+        "description": getattr(parameter, "description", None),
+        "id": parameter.name,
+        "metadata": parameter.metadata,
+    }
+    if parameter.file_path:
+        api_parameter["source"] = build_source_url(
+            parameter.file_path,
+            country_package_metadata,
+        )
+    if isinstance(parameter, Parameter):
+        if parameter.documentation:
+            api_parameter["documentation"] = parameter.documentation.strip()
+        api_parameter["values"] = build_api_values_history(parameter)
+    elif isinstance(parameter, Scale):
+        if "rate" in parameter.brackets[0].children:
+            api_parameter["brackets"] = build_api_scale(parameter, "rate")
+        elif "amount" in parameter.brackets[0].children:
+            api_parameter["brackets"] = build_api_scale(parameter, "amount")
+    elif isinstance(parameter, ParameterNode):
+        if parameter.documentation:
+            api_parameter["documentation"] = parameter.documentation.strip()
+        api_parameter["subparams"] = {
+            child_name: {
+                "description": child.description,
+            }
+            for child_name, child in parameter.children.items()
+        }
+    return api_parameter
+
+
+def build_parameters(tax_benefit_system, country_package_metadata):
+    return {
+        parameter.name.replace(".", "/"): build_api_parameter(
+            parameter,
+            country_package_metadata,
+        )
+        for parameter in tax_benefit_system.parameters.get_descendants()
+    }

openfisca_web_api/loader/spec.py

@@ -0,0 +1,168 @@
+import os
+from copy import deepcopy
+
+import dpath
+import yaml
+
+from openfisca_core.indexed_enums import Enum
+from openfisca_web_api import handlers
+
+OPEN_API_CONFIG_FILE = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)),
+    os.path.pardir,
+    "openAPI.yml",
+)
+
+
+def build_openAPI_specification(api_data):
+    tax_benefit_system = api_data["tax_benefit_system"]
+    with open(OPEN_API_CONFIG_FILE) as file:
+        spec = yaml.safe_load(file)
+    country_package_name = api_data["country_package_metadata"]["name"].title()
+    country_package_version = api_data["country_package_metadata"]["version"]
+    dpath.new(
+        spec,
+        "info/title",
+        spec["info"]["title"].replace("{COUNTRY_PACKAGE_NAME}", country_package_name),
+    )
+    dpath.new(
+        spec,
+        "info/description",
+        spec["info"]["description"].replace(
+            "{COUNTRY_PACKAGE_NAME}",
+            country_package_name,
+        ),
+    )
+    dpath.new(
+        spec,
+        "info/version",
+        spec["info"]["version"].replace(
+            "{COUNTRY_PACKAGE_VERSION}",
+            country_package_version,
+        ),
+    )
+
+    for entity in tax_benefit_system.entities:
+        name = entity.key.title()
+        spec["components"]["schemas"][name] = get_entity_json_schema(
+            entity,
+            tax_benefit_system,
+        )
+
+    situation_schema = get_situation_json_schema(tax_benefit_system)
+    dpath.new(spec, "components/schemas/SituationInput", situation_schema)
+    dpath.new(spec, "components/schemas/SituationOutput", situation_schema.copy())
+    dpath.new(
+        spec,
+        "components/schemas/Trace/properties/entitiesDescription/properties",
+        {
+            entity.plural: {"type": "array", "items": {"type": "string"}}
+            for entity in tax_benefit_system.entities
+        },
+    )
+
+    # Get example from the served tax benefist system
+
+    if tax_benefit_system.open_api_config.get("parameter_example"):
+        parameter_id = tax_benefit_system.open_api_config["parameter_example"]
+        parameter_path = parameter_id.replace(".", "/")
+        parameter_example = api_data["parameters"][parameter_path]
+    else:
+        parameter_example = next(iter(api_data["parameters"].values()))
+    dpath.new(spec, "components/schemas/Parameter/example", parameter_example)
+
+    if tax_benefit_system.open_api_config.get("variable_example"):
+        variable_example = api_data["variables"][
+            tax_benefit_system.open_api_config["variable_example"]
+        ]
+    else:
+        variable_example = next(iter(api_data["variables"].values()))
+    dpath.new(spec, "components/schemas/Variable/example", variable_example)
+
+    if tax_benefit_system.open_api_config.get("simulation_example"):
+        simulation_example = tax_benefit_system.open_api_config["simulation_example"]
+        dpath.new(
+            spec,
+            "components/schemas/SituationInput/example",
+            simulation_example,
+        )
+        dpath.new(
+            spec,
+            "components/schemas/SituationOutput/example",
+            handlers.calculate(tax_benefit_system, deepcopy(simulation_example)),
+        )  # calculate has side-effects
+        dpath.new(
+            spec,
+            "components/schemas/Trace/example",
+            handlers.trace(tax_benefit_system, simulation_example),
+        )
+    else:
+        message = f"No simulation example has been defined for this tax and benefit system. If you are the maintainer of {country_package_name}, you can define an example by following this documentation: https://openfisca.org/doc/openfisca-web-api/config-openapi.html"
+        dpath.new(spec, "components/schemas/SituationInput/example", message)
+        dpath.new(spec, "components/schemas/SituationOutput/example", message)
+        dpath.new(spec, "components/schemas/Trace/example", message)
+    return spec
+
+
+def get_variable_json_schema(variable):
+    result = {
+        "type": "object",
+        "additionalProperties": {"type": variable.json_type},
+    }
+
+    if variable.value_type == Enum:
+        result["additionalProperties"]["enum"] = [
+            item.name for item in list(variable.possible_values)
+        ]
+
+    return result
+
+
+def get_entity_json_schema(entity, tax_benefit_system):
+    if entity.is_person:
+        return {
+            "type": "object",
+            "properties": {
+                variable_name: get_variable_json_schema(variable)
+                for variable_name, variable in tax_benefit_system.get_variables(
+                    entity,
+                ).items()
+            },
+            "additionalProperties": False,
+        }
+    properties = {}
+    properties.update(
+        {
+            role.plural or role.key: {"type": "array", "items": {"type": "string"}}
+            for role in entity.roles
+        },
+    )
+    properties.update(
+        {
+            variable_name: get_variable_json_schema(variable)
+            for variable_name, variable in tax_benefit_system.get_variables(
+                entity,
+            ).items()
+        },
+    )
+    return {
+        "type": "object",
+        "properties": properties,
+        "additionalProperties": False,
+    }
+
+
+def get_situation_json_schema(tax_benefit_system):
+    return {
+        "type": "object",
+        "additionalProperties": False,
+        "properties": {
+            entity.plural: {
+                "type": "object",
+                "additionalProperties": {
+                    "$ref": f"#/components/schemas/{entity.key.title()}",
+                },
+            }
+            for entity in tax_benefit_system.entities
+        },
+    }