crimepy 0.1.0__py3-none-any.whl

crimepy/__init__.py

@@ -0,0 +1,25 @@
+"""Accessible tools for downloading, combining, munging, analyzing, and plotting crime data"""
+
+from __future__ import annotations
+
+__version__ = '0.1.0'
+
+__author__: str = 'Corey Rayburn Yung'
+
+__all__: list[str] = [
+    'Project',
+    'analyze',
+    'combine',
+    'nibrs',
+    'options',
+    'plot',
+    'techniques',
+    'utilities']
+
+# A study is an `amos.Project`, which crimepy offers under the same name.
+from amos import Project
+
+# Importing `nibrs`, `plot`, and `techniques` adds their techniques to the
+# `amos` library, so they can be named in settings as soon as crimepy is
+# imported.
+from . import analyze, combine, nibrs, options, plot, techniques, utilities

crimepy/analyze.py

@@ -0,0 +1,295 @@
+"""Analyses of crime data that researchers often need.
+
+Each function applies one of the `amos` techniques of `crimepy.techniques`,
+so the same analysis can be named in a project's settings. The functions
+take an `amos.Dataset` or a data frame with one row for each event (an
+incident, offense, victim, or arrest), such as those of `crimepy.nibrs.load`
+and `crimepy.nibrs.tables.build`:
+
+* `count`: the number of rows in each group and period, as a new dataset.
+* `trend`: counts over time, one column for each group.
+* `rates`: counts per 100,000 people (or another base).
+* `crime_rates`: the rate of crime in each group, using the population of the
+  agencies that reported for the whole year.
+* `clearance_rates`: the share of incidents cleared in each group.
+* `percent_change`: the change from one period to the next.
+* `crosstab`: counts (or shares) of two variables against each other.
+* `coverage`: how many agencies reported, and how many people they serve.
+
+Each function but `count` returns a table. Given a dataset, it also stores
+the table in the dataset's `tables`, under the technique's name (such as
+"clearance_rates").
+
+Contents:
+    clearance_rates: the share of incidents cleared in each group.
+    count: the number of rows in each group and period.
+    coverage: the agencies that reported and the people they serve.
+    crime_rates: crimes per 100,000 people, by group.
+    crosstab: counts or shares of two variables against each other.
+    percent_change: the change from one period to the next.
+    rates: counts per 100,000 people.
+    trend: counts over time, one column for each group.
+
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Any, Literal
+
+import amos
+import pandas as pd
+
+from . import techniques, utilities
+
+
+def count(
+    data: amos.Dataset | pd.DataFrame,
+    by: Sequence[str] | str | None = None,
+    *,
+    date: str | None = None,
+    freq: str | None = None,
+    weight: str | None = None,
+    name: str = 'count') -> amos.Dataset:
+    """Returns the number of rows in each group (and period).
+
+    Applies `crimepy.techniques.CountCrimes` to a copy of the data, so a
+    dataset that is passed keeps its rows.
+
+    Args:
+        data: one row for each event.
+        by: columns to group by. Defaults to `None`, which counts all rows
+            (in each period, if `freq` is set).
+        date: column of dates that `freq` groups by. Defaults to `None`, in
+            which case it is "incident_date" if `freq` is set.
+        freq: length of each period, as a `pandas` frequency ("YS" for years,
+            "QS" for quarters, "MS" for months, "W" for weeks, "D" for days).
+            Defaults to `None`, which does not group by period.
+        weight: column whose values are summed instead of counting rows.
+            Defaults to `None`.
+        name: name of the column of counts. Defaults to "count".
+
+    Returns:
+        A dataset with a column for each group (and "period") and the counts,
+            which are its label. Groups with no rows are left out.
+
+    """
+    return techniques.CountCrimes().apply(
+        utilities.to_dataset(utilities.to_frame(data)), label = name, by = by,
+        date = date, freq = freq, weight = weight, name = name)
+
+
+def trend(
+    data: amos.Dataset | pd.DataFrame,
+    by: str | None = None,
+    *,
+    date: str = 'incident_date',
+    freq: str = 'MS',
+    top: int | None = None) -> pd.DataFrame:
+    """Returns counts over time, with one column for each group.
+
+    Args:
+        data: one row for each event.
+        by: column whose values become columns. Defaults to `None`, which
+            makes one column, "count".
+        date: column of dates. Defaults to "incident_date".
+        freq: length of each period, as a `pandas` frequency. Defaults to
+            "MS" (months).
+        top: keep only the groups with the most rows. Defaults to `None`,
+            which keeps them all.
+
+    Returns:
+        A table indexed by the start of each period, with a column of counts
+            for each group (0 in periods with none).
+
+    """
+    return _describe(
+        techniques.CrimeTrend(), data, by = by, date = date, freq = freq,
+        top = top)
+
+
+def rates(
+    counts: amos.Dataset | pd.DataFrame,
+    population: amos.Dataset | pd.DataFrame | str,
+    *,
+    on: Sequence[str] | str | None = None,
+    count: str = 'count',
+    per: float = 100_000,
+    name: str = 'rate') -> pd.DataFrame:
+    """Returns counts per `per` people.
+
+    Args:
+        counts: a table of counts (such as one from `count`).
+        population: a column of `counts` with the population, or a table of
+            populations to join to `counts` on `on`, with a "population"
+            column.
+        on: the columns that join `population` to `counts`. Defaults to
+            `None`, in which case they are the columns that both have (other
+            than "population").
+        count: the column of counts. Defaults to "count".
+        per: the number of people that rates are per. Defaults to 100,000.
+        name: name of the column of rates. Defaults to "rate".
+
+    Returns:
+        `counts`, with the population (if it was joined) and the rates
+            (missing where the population is 0 or missing).
+
+    """
+    return _describe(
+        techniques.PopulationRates(), counts, population = population, on = on,
+        count = count, per = per, name = name)
+
+
+def crime_rates(
+    events: amos.Dataset | pd.DataFrame,
+    agencies: amos.Dataset | pd.DataFrame | None = None,
+    by: Sequence[str] | str = ('data_year', 'state'),
+    *,
+    months: int = 12,
+    per: float = 100_000,
+    folder: Any = None) -> pd.DataFrame:
+    """Returns the number of events per `per` people in each group.
+
+    Only agencies that reported in at least `months` months of the year (see
+    "months_active" in `crimepy.nibrs.munge`) count, both for the events and
+    for the population, so that the rate is not lowered by agencies that did
+    not report.
+
+    Args:
+        events: one row for each event (such as an incident), with
+            "data_year" and "agency_ori" (or "ori").
+        agencies: the agencies, with "data_year", "ori", "population", and
+            "months_active". Defaults to `None`, in which case the agencies
+            of the years (and states) of `events` are loaded.
+        by: columns of the agencies to group by. Defaults to ("data_year",
+            "state").
+        months: the fewest months an agency must have reported in. Defaults
+            to 12.
+        per: the number of people that rates are per. Defaults to 100,000.
+        folder: the data folder to load agencies from. Defaults to `None`, in
+            which case it is `crimepy.options.DATA_FOLDER`.
+
+    Returns:
+        A table with a column for each group, "count", "population",
+            "agencies", and "rate".
+
+    """
+    return _describe(
+        techniques.CrimeRates(), events, agencies = agencies, by = by,
+        months = months, per = per, folder = folder)
+
+
+def clearance_rates(
+    incidents: amos.Dataset | pd.DataFrame,
+    by: Sequence[str] | str | None = None,
+    *,
+    cleared: str = 'cleared') -> pd.DataFrame:
+    """Returns the share of incidents that were cleared in each group.
+
+    Args:
+        incidents: one row for each incident, with a column saying whether
+            it was cleared (such as from `crimepy.nibrs.tables.incidents`).
+        by: columns to group by. Defaults to `None`, which uses all rows.
+        cleared: the column saying whether each incident was cleared.
+            Defaults to "cleared".
+
+    Returns:
+        A table with a column for each group, "incidents", "cleared", and
+            "clearance_rate" (from 0 to 1).
+
+    """
+    return _describe(
+        techniques.ClearanceRates(), incidents, by = by, cleared = cleared)
+
+
+def percent_change(
+    data: amos.Dataset | pd.DataFrame | pd.Series,
+    periods: int = 1) -> pd.DataFrame | pd.Series:
+    """Returns the percent change from one period to the next.
+
+    Args:
+        data: counts (or rates) in each period, in order, such as a table
+            from `trend`.
+        periods: the number of periods to compare across (12 compares each
+            month with the same month of the year before). Defaults to 1.
+
+    Returns:
+        The change, in percent, with the index of `data`.
+
+    """
+    if isinstance(data, pd.Series):
+        return data.pct_change(periods = periods, fill_method = None) * 100
+    return _describe(techniques.PercentChange(), data, periods = periods)
+
+
+def crosstab(
+    data: amos.Dataset | pd.DataFrame,
+    rows: str,
+    columns: str,
+    *,
+    normalize: bool | Literal['all', 'index', 'columns'] = False,
+    margins: bool = False) -> pd.DataFrame:
+    """Returns counts (or shares) of two variables against each other.
+
+    Args:
+        data: one row for each event.
+        rows: the column whose values become rows.
+        columns: the column whose values become columns.
+        normalize: False for counts, True for shares of all rows, "index"
+            for shares of each row, or "columns" for shares of each column.
+            Defaults to `False`.
+        margins: whether to add totals. Defaults to `False`.
+
+    Returns:
+        The table.
+
+    """
+    return _describe(
+        techniques.Crosstab(), data, rows = rows, columns = columns,
+        normalize = normalize, margins = margins)
+
+
+def coverage(
+    agencies: amos.Dataset | pd.DataFrame,
+    by: Sequence[str] | str = ('data_year', 'state'),
+    *,
+    months: int = 12) -> pd.DataFrame:
+    """Returns how many agencies reported and how many people they serve.
+
+    Agencies covered by another agency (whose crimes the other agency
+    reports) are left out, so that people are not counted twice.
+
+    Args:
+        agencies: the agencies, with "population" and "months_active".
+        by: columns to group by. Defaults to ("data_year", "state").
+        months: the months that an agency must have reported in to count as
+            reporting for the year. Defaults to 12.
+
+    Returns:
+        A table with a column for each group, "agencies", "reporting" (the
+            agencies that reported in at least `months` months),
+            "population", "population_reporting", and "share_reporting" (the
+            share of the population served by agencies that reported).
+
+    """
+    return _describe(techniques.Coverage(), agencies, by = by, months = months)
+
+
+def _describe(
+    technique: amos.Describer,
+    data: amos.Dataset | pd.DataFrame,
+    **kwargs: Any) -> pd.DataFrame:
+    """Applies a describer to `data` and returns its table.
+
+    Args:
+        technique: the describer.
+        data: a dataset (which keeps the table in its `tables`) or a data
+            frame.
+        **kwargs: parameters for the describer.
+
+    Returns:
+        The table.
+
+    """
+    dataset = technique.apply(utilities.to_dataset(data), **kwargs)
+    return dataset.tables[technique.name]

crimepy/cli.py

@@ -0,0 +1,185 @@
+"""The command line interface of crimepy.
+
+```sh
+crimepy nibrs years
+crimepy nibrs download 2021-2023
+crimepy nibrs convert 2021-2023
+crimepy nibrs export victims 2022 --states DE --output victims.csv
+crimepy project study.ini --export
+```
+
+`export` writes a table of the master files (such as "agencies") or a built
+table (such as "incidents"; see `crimepy.nibrs.tables`) to a csv or parquet
+file, downloading and converting the years first if needed. `project` runs an
+`amos` project from a settings file.
+
+Contents:
+    main: runs the command line interface.
+
+"""
+
+from __future__ import annotations
+
+import argparse
+import pathlib
+import sys
+from collections.abc import Sequence
+
+from . import nibrs, options
+
+
+def main(arguments: Sequence[str] | None = None) -> int:
+    """Runs the command line interface.
+
+    Args:
+        arguments: the arguments, without the program's name. Defaults to
+            `None`, in which case they are those that the program was called
+            with.
+
+    Returns:
+        The exit status: 0 if the command succeeded.
+
+    """
+    parser = _parser()
+    parsed = parser.parse_args(arguments)
+    if not hasattr(parsed, 'run'):
+        parser.print_help()
+        return 1
+    return int(parsed.run(parsed) or 0)
+
+
+def _parser() -> argparse.ArgumentParser:
+    """Returns the parser of the command line arguments."""
+    parser = argparse.ArgumentParser(
+        prog = 'crimepy',
+        description = (
+            'Download, combine, munge, analyze, and plot crime data.'))
+    commands = parser.add_subparsers(title = 'commands')
+    data = commands.add_parser('nibrs', help = 'work with NIBRS master files')
+    tasks = data.add_subparsers(title = 'NIBRS commands')
+
+    years = tasks.add_parser('years', help = 'list the years the FBI has')
+    years.set_defaults(run = _years)
+
+    download = tasks.add_parser('download', help = 'download master files')
+    _add_years(download)
+    _add_folder(download)
+    download.add_argument(
+        '--overwrite', action = 'store_true',
+        help = 'download files that were downloaded before')
+    download.set_defaults(run = _download)
+
+    convert = tasks.add_parser(
+        'convert', help = 'convert master files to parquet files')
+    _add_years(convert)
+    _add_folder(convert)
+    convert.add_argument(
+        '--tables', nargs = '+', default = None,
+        help = 'the tables to convert (all of them by default)')
+    convert.add_argument(
+        '--overwrite', action = 'store_true',
+        help = 'convert tables that were converted before')
+    convert.set_defaults(run = _convert)
+
+    export = tasks.add_parser(
+        'export', help = 'write a table to a csv or parquet file')
+    export.add_argument(
+        'table',
+        choices = sorted({*nibrs.layouts.SEGMENTS, *nibrs.tables.BUILDERS}),
+        help = 'the table to write')
+    _add_years(export)
+    _add_folder(export)
+    export.add_argument(
+        '--states', nargs = '+', default = None,
+        help = 'postal abbreviations of the states to keep')
+    export.add_argument(
+        '--codes', action = 'store_true',
+        help = 'keep codes instead of replacing them with labels')
+    export.add_argument(
+        '--output', required = True, type = pathlib.Path,
+        help = 'the file to write (.csv or .parquet)')
+    export.set_defaults(run = _export)
+
+    project = commands.add_parser(
+        'project', help = 'run an amos project from a settings file')
+    project.add_argument('settings', type = pathlib.Path)
+    project.add_argument(
+        '--export', action = 'store_true',
+        help = 'export the results to a folder named for the run')
+    project.set_defaults(run = _project)
+    return parser
+
+
+def _add_folder(parser: argparse.ArgumentParser) -> None:
+    """Adds the "--folder" argument to `parser`."""
+    parser.add_argument(
+        '--folder', type = pathlib.Path, default = None,
+        help = f'the data folder (by default, {options.DATA_FOLDER})')
+
+
+def _add_years(parser: argparse.ArgumentParser) -> None:
+    """Adds the "years" argument to `parser`."""
+    parser.add_argument(
+        'years', nargs = '+',
+        help = 'years or ranges of years, such as 2022 or 2015-2023')
+
+
+def _convert(arguments: argparse.Namespace) -> int:
+    """Converts master files to parquet files."""
+    converted = nibrs.convert(
+        arguments.years, arguments.folder, tables = arguments.tables,
+        overwrite = arguments.overwrite)
+    for year, paths in converted.items():
+        folder = next(iter(paths.values())).parent
+        print(f'{year}: {len(paths)} tables in {folder}')
+    return 0
+
+
+def _download(arguments: argparse.Namespace) -> int:
+    """Downloads master files."""
+    for path in nibrs.download(
+        arguments.years, arguments.folder, overwrite = arguments.overwrite):
+        print(path)
+    return 0
+
+
+def _export(arguments: argparse.Namespace) -> int:
+    """Writes a table to a csv or parquet file."""
+    suffix = arguments.output.suffix.lower()
+    if suffix not in ('.csv', '.parquet'):
+        print('the output must be a .csv or .parquet file', file = sys.stderr)
+        return 2
+    decode = not arguments.codes
+    if arguments.table in nibrs.tables.BUILDERS:
+        data = nibrs.tables.build(
+            arguments.table, arguments.years, states = arguments.states,
+            folder = arguments.folder, decode = decode).data
+    else:
+        data = nibrs.load_frame(
+            arguments.table, arguments.years, states = arguments.states,
+            folder = arguments.folder, decode = decode)
+    arguments.output.parent.mkdir(parents = True, exist_ok = True)
+    if suffix == '.csv':
+        data.to_csv(arguments.output, index = False)
+    else:
+        data.to_parquet(arguments.output, index = False)
+    print(f'wrote {len(data):,} rows to {arguments.output}')
+    return 0
+
+
+def _project(arguments: argparse.Namespace) -> int:
+    """Runs an amos project from a settings file."""
+    import amos  # noqa: PLC0415
+
+    project = amos.Project.create(arguments.settings)
+    if arguments.export:
+        print(project.export())
+    print(project.result)
+    return 0
+
+
+def _years(arguments: argparse.Namespace) -> int:  # noqa: ARG001
+    """Lists the years of NIBRS master files that the FBI has."""
+    found = nibrs.available_years()
+    print(f'{found[0]}-{found[-1]}')
+    return 0

crimepy/combine.py

@@ -0,0 +1,154 @@
+"""Combines tables of crime data.
+
+`pandas.concat` turns a column of categories into plain objects when the
+tables have different categories (as tables of different years usually do),
+and `pandas.merge` does the same to the keys it joins on. These functions
+keep the categories, so combined tables stay small and keep their labels.
+
+They take `amos` datasets or data frames and return an `amos.Dataset`, which
+keeps the label, task, and groups of the first dataset given when its
+columns are still there.
+
+Contents:
+    merge: joins two tables, keeping their categories.
+    stack: stacks tables on top of each other, keeping their categories.
+
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Sequence
+from typing import Any
+
+import amos
+import pandas as pd
+from pandas.api.types import union_categoricals
+
+from . import utilities
+
+
+def stack(
+    tables: Iterable[amos.Dataset | pd.DataFrame],
+    *,
+    source: str | None = None,
+    labels: Sequence[Any] | None = None) -> amos.Dataset:
+    """Stacks tables on top of each other, keeping their categories.
+
+    Columns that are in some tables but not others are missing (`NA`) in the
+    rows of the others. A column of categories in every table that has it
+    stays a column of categories, with all of their categories.
+
+    Args:
+        tables: the tables to stack (such as one table for each year).
+        source: name of a column to add that says which table each row came
+            from. Defaults to `None`, which adds no column.
+        labels: what `source` says for each table. Defaults to `None`, in
+            which case the tables are numbered from 0.
+
+    Raises:
+        ValueError: if there are no tables, or `labels` does not have one
+            label for each table.
+
+    Returns:
+        A dataset of the stacked table, with a new index.
+
+    """
+    items = list(tables)
+    frames = [utilities.to_frame(t) for t in items]
+    if not frames:
+        message = 'there are no tables to stack'
+        raise ValueError(message)
+    if labels is not None and len(labels) != len(frames):
+        message = 'labels must have one label for each table'
+        raise ValueError(message)
+    if source is not None:
+        names = list(labels) if labels is not None else list(range(len(frames)))
+        frames = [
+            f.assign(**{source: name}) for f, name in zip(frames, names, strict = True)]
+    columns = list(dict.fromkeys(c for f in frames for c in f.columns))
+    categorical = [
+        c for c in columns
+        if all(
+            isinstance(f[c].dtype, pd.CategoricalDtype)
+            for f in frames if c in f.columns)]
+    for column in categorical:
+        union = union_categoricals(
+            [f[column].array for f in frames if column in f.columns],
+            ignore_order = True)
+        frames = [
+            f.assign(**{column: f[column].cat.set_categories(union.categories)})
+            if column in f.columns else f
+            for f in frames]
+    stacked = pd.concat(frames, ignore_index = True)
+    for column in categorical:
+        if not isinstance(stacked[column].dtype, pd.CategoricalDtype):
+            stacked[column] = stacked[column].astype('category')
+    return _dataset(stacked, items)
+
+
+def merge(
+    left: amos.Dataset | pd.DataFrame,
+    right: amos.Dataset | pd.DataFrame,
+    on: Sequence[str] | str,
+    *,
+    how: str = 'left',
+    suffixes: tuple[str, str] = ('', '_right'),
+    validate: str | None = None) -> amos.Dataset:
+    """Joins two tables, keeping the categories of their columns.
+
+    Args:
+        left: the first table.
+        right: the second table.
+        on: the columns to join on, which both tables have.
+        how: "left", "right", "inner", or "outer". Defaults to "left".
+        suffixes: added to the names of columns (other than `on`) that both
+            tables have: the first to those of `left`, the second to those of
+            `right`. Defaults to ('', '_right').
+        validate: checks the kind of join, as in `pandas.merge` (such as
+            "many_to_one"). Defaults to `None`, which checks nothing.
+
+    Returns:
+        A dataset of the joined table.
+
+    """
+    keys = [on] if isinstance(on, str) else list(on)
+    restore = {}
+    items = [left, right]
+    left, right = utilities.to_frame(left).copy(), utilities.to_frame(right).copy()
+    for key in keys:
+        kinds = (left[key].dtype, right[key].dtype)
+        if any(isinstance(k, pd.CategoricalDtype) for k in kinds):
+            restore[key] = True
+            left[key] = left[key].astype('object')
+            right[key] = right[key].astype('object')
+    joined = left.merge(
+        right, on = keys, how = how, suffixes = suffixes,  # type: ignore[arg-type]
+        validate = validate)  # type: ignore[arg-type]
+    for key in restore:
+        joined[key] = joined[key].astype('category')
+    return _dataset(joined, items)
+
+
+def _dataset(
+    data: pd.DataFrame,
+    items: Sequence[amos.Dataset | pd.DataFrame]) -> amos.Dataset:
+    """Returns `data` as a dataset like the first dataset of `items`.
+
+    Args:
+        data: the combined table.
+        items: the datasets and data frames that were combined.
+
+    Returns:
+        A dataset with the label, task, and groups of the first dataset in
+            `items` whose label and groups are columns of `data` (and with
+            none if no dataset was given).
+
+    """
+    first = next((i for i in items if isinstance(i, amos.Dataset)), None)
+    if first is None:
+        return amos.Dataset.create(data)
+    label = first.label if first.label in data.columns else None
+    groups = [g for g in first.groups if g in data.columns]
+    return amos.Dataset.create(
+        data, label = label, task = first.task if label else None,
+        seed = first.seed, groups = groups or None)