Package {cellsynth}

Force roles onto a spec by column name or name pattern

Description

Used by infer_spec() and by synthesize_file() (where it also applies to a reviewed spec read from disk, so identifiers named in the call always win). Names match the internal or the on-disk column name, ignoring case; patterns are regular expressions (perl, case-insensitive) matched against the on-disk name. The options cellsynth.id, cellsynth.id_pattern, cellsynth.text and cellsynth.text_pattern add session-wide defaults, e.g. options(cellsynth.id_pattern = "^(pat|subj)") once per session.

Usage

apply_role_overrides(
  spec,
  id = NULL,
  text = NULL,
  roles = NULL,
  id_pattern = NULL,
  text_pattern = NULL,
  warn_unknown = TRUE
)

Arguments

Value

The spec with roles replaced and reason set to "user: ...".

Examples

df <- data.frame(PatKey = sprintf("%04d", 1:60), Visit_Note = "seen",
                 hb = round(seq(110, 160, length.out = 60)))
path <- file.path(tempdir(), "overrides.csv")
utils::write.csv(df, path, row.names = FALSE)
spec <- infer_spec(read_source(path))
spec <- apply_role_overrides(spec, id = "patkey", text_pattern = "note$")
spec[, c("orig_name", "role", "reason")]

Build the predictor frame for one round

Description

Build the predictor frame for one round

Usage

build_predictors(raw, roles, censor = NULL, lump = 15)

Arguments

Value

list(P = data.frame with syntactic names, group = index into raw columns for every column of P).

Mean absolute difference between correlation matrices

Description

Over the numeric, date and binary categorical columns, Spearman correlations on pairwise complete rows.

Usage

compare_correlations(orig, syn, spec)

Arguments

Value

list(mad = mean absolute difference, orig, syn matrices).

Examples

df <- data.frame(id = sprintf("AB%03d", 1:60), value = 1:60,
                 code = rep(c(0, 1, NA), 20))
path <- file.path(tempdir(), "compare_correlations.csv")
utils::write.csv(df, path, row.names = FALSE, na = "")
src <- read_source(path)
spec <- infer_spec(src)
syn <- src$raw
syn$value <- rev(syn$value)
compare_correlations(src$raw, syn, spec)

Compare original and synthetic tables column by column

Description

For every modelled column: missing fraction in both; for numeric and date columns the mean, SD and median and the Kolmogorov-Smirnov distance; for categorical columns the total variation distance between label distributions. self_match is the share of subject cells equal to the same row's original cell; for categorical columns chance_match gives what a random redraw from the marginal would produce, so a large excess points at a column that copies its own row.

Usage

compare_tables(orig, syn, spec, rows = seq_len(nrow(orig)))

Arguments

Value

A data.frame, one row per modelled column.

Examples

df <- data.frame(id = sprintf("AB%03d", 1:60), value = 1:60,
                 code = rep(c(0, 1, NA), 20), group = rep(c("A", "B"), 30))
path <- file.path(tempdir(), "compare_tables.csv")
utils::write.csv(df, path, row.names = FALSE, na = "")
src <- read_source(path)
spec <- infer_spec(src)
syn <- src$raw
syn$value <- rev(syn$value)
compare_tables(src$raw, syn, spec)

Detect rows that are not subjects (totals, site headers)

Description

A row is non-subject when any identifier cell reads like a summary label (TOTAL, Sum, Osszesen with or without accent), or when the main identifier carries no digit while at least 90% of the others do (site-name header rows).

Usage

detect_nonsubject_rows(raw, spec)

Arguments

Value

Logical vector, TRUE for non-subject rows.

Examples

df <- data.frame(id = c(sprintf("AB%03d", 1:59), "TOTAL"), value = 1:60)
path <- file.path(tempdir(), "nonsubject_rows.csv")
utils::write.csv(df, path, row.names = FALSE, na = "")
src <- read_source(path)
spec <- infer_spec(src, id = "id")
flagged <- detect_nonsubject_rows(src$raw, spec)
src$raw[flagged, , drop = FALSE]

Infer a column specification for a source table

Description

Heuristics, in order: all-missing columns are empty; trial and EDC key names (USUBJID, SUBJID, SITEID, STUDYID, RANDNO, ...) are id; identifier-like names with near-unique values (or code-like values such as AB123) are id; a single value with no missing cells is constant; mostly-date columns are date; mostly-numeric columns are numeric, or categorical if they take few distinct values (coded 0/1/2 fields); remaining character columns are categorical when the label set is small and text otherwise.

Usage

infer_spec(
  src,
  id = NULL,
  text = NULL,
  roles = NULL,
  id_pattern = NULL,
  text_pattern = NULL,
  max_levels = 10,
  max_cat_levels = 60,
  warn_unknown = TRUE
)

Arguments

Value

A data.frame, one row per column: column, orig_name, role, n_obs, n_unique, miss_frac, num_frac, date_frac, reason.

Examples

df <- data.frame(id = sprintf("AB%03d", 1:60), value = 1:60,
                 code = rep(c(0, 1, NA), 20), group = rep(c("A", "B"), 30))
path <- file.path(tempdir(), "infer_spec.csv")
utils::write.csv(df, path, row.names = FALSE, na = "")
src <- read_source(path)
spec <- infer_spec(src, id = "id", roles = c(code = "categorical"))
spec[, c("column", "role", "reason")]

Draw donor rows from CART leaves

Description

Fits y ~ X on the training rows and, for every row to predict, returns a training row drawn at random from the same leaf.

Usage

leaf_donors(y, X, train, pred, minbucket = 5, cp = 1e-04)

Arguments

Value

list(donor = row indices into y, leaves, fallback flag).

Align columns of another table as auxiliary predictors

Description

Auxiliary columns condition the synthesis (so cross-table associations survive a later join) but are never censored or written out.

Usage

make_aux(
  src,
  by,
  aux_src,
  aux_spec,
  aux_by,
  cols = NULL,
  key_fun = function(k) toupper(gsub("[^A-Za-z0-9]", "", k))
)

Arguments

Value

list(raw, roles), rows aligned to src$raw (first match per key).

Examples

df <- data.frame(id = sprintf("AB%03d", 1:60), value = 1:60)
other <- data.frame(id = rev(df$id), code = rep(c(0, 1, NA), 20))
path <- file.path(tempdir(), "make_aux_main.csv")
aux_path <- file.path(tempdir(), "make_aux_other.csv")
utils::write.csv(df, path, row.names = FALSE, na = "")
utils::write.csv(other, aux_path, row.names = FALSE, na = "")
src <- read_source(path)
aux_src <- read_source(aux_path)
aux_spec <- infer_spec(aux_src)
aux <- make_aux(src, by = "id", aux_src = aux_src,
                aux_spec = aux_spec, aux_by = "id", cols = "code")
head(aux$raw)

Normalise categorical labels for modelling

Description

Numeric-looking labels collapse ("1.0", "1", " 1" -> "1"); other labels are trimmed. The raw strings are still what the output copies.

Usage

normalise_label(x)

Arguments

Value

Character vector of modelling labels.

Parse raw strings as dates (days since 1970-01-01)

Description

Recognises ISO yyyy-mm-dd (with optional time), yyyy.mm.dd., yyyy/mm/dd, dd.mm.yyyy and, when allow_serial = TRUE, Excel date serials (numbers between 3000 and 73000, i.e. 1908 to 2099).

Usage

parse_date_chr(x, allow_serial = FALSE)

Arguments

Value

Numeric vector of days since 1970-01-01, NA if not a date.

Examples

x <- c("2020-01-15", "16.01.2020", "2020/01/17", "43848", NA)
days <- parse_date_chr(x)
as.Date(days, origin = "1970-01-01")
days <- parse_date_chr(x, allow_serial = TRUE)
as.Date(days, origin = "1970-01-01")

Parse raw strings as numbers, tolerating decimal commas

Description

Parse raw strings as numbers, tolerating decimal commas

Usage

parse_number_chr(x)

Arguments

Value

Numeric vector, NA where a cell is not a number.

Examples

x <- c("1.5", "2,75", " 3 ", "1e2", "unknown", NA)
parsed <- parse_number_chr(x)
parsed
is.na(parsed)
data.frame(raw = x, numeric = parsed)

A result that is not synthesised: copied, blanked or dropped

Description

For sheets that are empty, have no modelled columns, or are too small to synthesise (fewer than 4 * minbucket subject rows): legends, codebooks, lookup tables, tiny pilot tables.

Usage

passthrough_result(src, spec, how = c("blank", "copy", "drop"), reason = "")

Arguments

Value

A synth_result with settings$passthrough set.

Examples

df <- data.frame(id = sprintf("AB%03d", 1:6), value = 1:6)
path <- file.path(tempdir(), "passthrough_result.csv")
utils::write.csv(df, path, row.names = FALSE, na = "")
src <- read_source(path)
spec <- infer_spec(src)
res <- passthrough_result(src, spec, how = "blank", reason = "Tiny table")
res$synthetic

Print a synth_result

Description

Print a synth_result

Usage

## S3 method for class 'synth_result'
print(x, ...)

Arguments

Value

x, invisibly.

Print a synth_source

Description

Print a synth_source

Usage

## S3 method for class 'synth_source'
print(x, ...)

Arguments

Value

x, invisibly.

Read a source table (xlsx or csv) as raw character data

Description

The first row is always taken as the header. Excel formulas are read as their cached values; cell formatting (colours, merged cells, number formats) is not read. CSV files are read with an auto-detected delimiter (comma, semicolon, tab or pipe) and encoding (Hungarian Excel exports are often semicolon-separated windows-1250).

Usage

read_source(
  path,
  sheet = NULL,
  guess_max = 1e+05,
  delim = NULL,
  encoding = NULL
)

Arguments

Value

A synth_source list: raw (data.frame of character, internal unique names), orig_names (header as on disk, duplicates allowed), orig_class (readxl/readr guessed class per column), path, sheet, format, and for CSV delim and encoding.

Examples

df <- data.frame(id = sprintf("AB%03d", 1:60), value = 1:60,
                 code = rep(c(0, 1, NA), 20))
path <- file.path(tempdir(), "read_source.csv")
utils::write.csv(df, path, row.names = FALSE, na = "")
src <- read_source(path)
head(src$raw)

Read a spec CSV written by write_spec()

Description

Read a spec CSV written by write_spec()

Usage

read_spec(path)

Arguments

Value

A spec data.frame.

Examples

df <- data.frame(id = sprintf("AB%03d", 1:60), value = 1:60)
path <- file.path(tempdir(), "read_spec_source.csv")
utils::write.csv(df, path, row.names = FALSE, na = "")
spec <- infer_spec(read_source(path))
spec_path <- file.path(tempdir(), "read_spec.csv")
write_spec(spec, spec_path)
reviewed <- read_spec(spec_path)
reviewed[, c("column", "role")]

Share of synthetic rows identical to their own original row

Description

Compares the modelled columns only (identifiers are equal by design).

Usage

row_disclosure(orig, syn, spec, rows = seq_len(nrow(orig)), min_share = 0.9)

Arguments

Value

list(identical = share of rows fully identical, identical_unique = share of rows identical AND unique in the original (the disclosive kind), near = share of rows matching at least min_share of their cells, per_row = vector).

Examples

df <- data.frame(id = sprintf("AB%03d", 1:60), value = 1:60,
                 code = rep(c(0, 1, NA), 20))
path <- file.path(tempdir(), "row_disclosure.csv")
utils::write.csv(df, path, row.names = FALSE, na = "")
src <- read_source(path)
spec <- infer_spec(src)
syn <- src$raw
syn$value[1:30] <- rev(syn$value[1:30])
row_disclosure(src$raw, syn, spec, min_share = 0.5)

Choose predictors for every target column by association screening

Description

Association between two columns is the largest absolute Pearson correlation between any of their screening signals (values, missingness indicators, level dummies). Screening runs once, on the original table.

Usage

screen_predictors(
  raw,
  roles,
  targets,
  keep = 100,
  lock_threshold = 0.95,
  block_cells = 2e+07
)

Arguments

Details

Memory use is bounded for very wide tables: the column x column score is computed in blocks of columns and only each target's keep best candidates and the near-deterministic pairs are retained, never the full matrix (a 9,000-column sheet would need about 0.7 GB for it).

Value

List, for each target, the indices of its candidate predictor columns, best first; attribute lock_pairs is a two-column matrix of target index pairs with score >= lock_threshold.

Sheet names of an Excel workbook

Description

Sheet names of an Excel workbook

Usage

source_sheets(path)

Arguments

Value

Character vector, in workbook order.

Examples

df <- data.frame(id = sprintf("AB%03d", 1:6), value = 1:6)
path <- file.path(tempdir(), "source_sheets.xlsx")
sheets <- list(Subjects = df, Codes = data.frame(code = c(0, 1)))
openxlsx::write.xlsx(sheets, path, overwrite = TRUE)
source_sheets(path)

Synthesise a file end to end

Description

Reads the source (every sheet of a workbook unless sheet says otherwise), infers a spec per sheet (or reads the reviewed one), writes inferred specs for review, synthesises m independent copies and writes each as ⁠<stem>_synth<i>.<ext>⁠ (all sheets in one workbook, in the original order) with a run log.

Usage

synthesize_file(
  path,
  out_dir,
  sheet = NULL,
  spec_path = NULL,
  spec_dir = out_dir,
  id = NULL,
  text = NULL,
  id_pattern = NULL,
  text_pattern = NULL,
  m = 1,
  seed = 1,
  small_sheets = c("blank", "copy", "drop"),
  minbucket = 5,
  delim = NULL,
  encoding = NULL,
  ...
)

Arguments

Details

Review the spec CSVs after the first run: set role to id for every identifier the heuristics missed and to text for free text, then run again with the same spec_dir (or spec_path).

Sheets that cannot be synthesised are handled by small_sheets: empty sheets and sheets without modelled columns are copied as they are (they hold no modelled values); sheets with fewer than 4 * minbucket subject rows (legends, codebooks, lookup tables, tiny pilots) get small_sheets: "blank" (default; keeps header and identifiers, empties the rest), "copy" (verbatim, for codebooks you know hold no subject data) or "drop".

Value

list(results = list over copies of lists over sheets of synth_result, files, spec_paths).

Examples

df <- data.frame(id = sprintf("AB%03d", 1:60), value = 1:60,
                 code = rep(c(0, 1, NA), 20), group = rep(c("A", "B"), 30))
path <- file.path(tempdir(), "synthesize_file.csv")
utils::write.csv(df, path, row.names = FALSE, na = "")
out <- synthesize_file(path, out_dir = file.path(tempdir(), "synth_output"),
                       m = 1, seed = 1, n_folds = 2, verbose = FALSE)
out$files
out$spec_paths
head(out$results[[1]][[1]]$synthetic)

Synthesise one table

Description

Synthesise one table

Usage

synthesize_table(
  src,
  spec = NULL,
  n_folds = 10,
  n_cycles = 1,
  censor_mode = c("cell", "row"),
  lock_threshold = 0.95,
  max_predictors = 25,
  minbucket = 5,
  cp = 1e-04,
  aux = NULL,
  aux_always = TRUE,
  nonsubject = c("blank", "keep"),
  seed = NULL,
  verbose = TRUE
)

Arguments

Value

A synth_result: synthetic (raw character data.frame in the source layout), spec, log, nonsubject rows, settings, and the source metadata needed to write it out.

Examples

df <- data.frame(id = sprintf("AB%03d", 1:60), value = 1:60,
                 code = rep(c(0, 1, NA), 20), group = rep(c("A", "B"), 30))
path <- file.path(tempdir(), "synthesize_table.csv")
utils::write.csv(df, path, row.names = FALSE, na = "")
src <- read_source(path)
res <- synthesize_table(src, n_folds = 2, seed = 1, verbose = FALSE)
head(res$synthetic)

Check a specification against its source

Description

Check a specification against its source

Usage

validate_spec(spec, src)

Arguments

Value

spec, invisibly; stops on a mismatch.

Write a spec to CSV for review and editing

Description

Only the role column is meant to be edited by hand; the rest documents why the heuristics chose it. The file holds column names and counts only, no cell values.

Usage

write_spec(spec, path)

Arguments

Value

path, invisibly.

Examples

df <- data.frame(id = sprintf("AB%03d", 1:60), value = 1:60)
path <- file.path(tempdir(), "write_spec_source.csv")
utils::write.csv(df, path, row.names = FALSE, na = "")
spec <- infer_spec(read_source(path))
spec_path <- file.path(tempdir(), "write_spec.csv")
write_spec(spec, spec_path)
file.exists(spec_path)

Write a synthetic table (or workbook) in the format of its source

Description

Excel output keeps the original header (duplicate names included), the sheet names and order and, column by column, the cell type readxl saw in the source (numbers as numbers, dates as dates, everything else as text). Formulas, formatting and merged cells are not reproduced (values only). .xls sources are written as .xlsx. CSV output writes the raw strings with the original header and delimiter, in UTF-8.

Usage

write_synthetic(res, path)

Arguments

Value

path, invisibly.

Examples

df <- data.frame(id = sprintf("AB%03d", 1:60), value = 1:60,
                 code = rep(c(0, 1, NA), 20))
path <- file.path(tempdir(), "write_synthetic_source.csv")
utils::write.csv(df, path, row.names = FALSE, na = "")
src <- read_source(path)
res <- synthesize_table(src, n_folds = 2, seed = 1, verbose = FALSE)
out_path <- file.path(tempdir(), "write_synthetic_output.csv")
write_synthetic(res, out_path)
file.exists(out_path)