Skip to content

API: preprocessing.inspection

skyulf.preprocessing.inspection

Inspection nodes (DatasetProfile, DataSnapshot).

Both nodes are read-only; the appliers are pure passthroughs. Only the fits collect engine-specific summary statistics, so they route through :func:fit_dual_engine.

DataSnapshotApplier

Bases: BaseApplier

Passthrough applier; the snapshot is produced entirely at fit time.

Source code in skyulf-core/skyulf/preprocessing/inspection.py 
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
class DataSnapshotApplier(BaseApplier):
    """Passthrough applier; the snapshot is produced entirely at fit time."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Check ownership without interpreting or recapturing training values."""
        local_state_fields(raw, "data_snapshot", {"type", "snapshot"})
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe the existing passthrough without taking another snapshot."""
        if engine not in {"pandas", "polars"}:
            return None
        DataSnapshotApplier.validate_inference_state(state)
        return ExecutionCapability(engine, "apply", "local", "preserve", "row")

    def apply(
        self,
        df: pd.DataFrame | SkyulfDataFrame | tuple[Any, ...] | Any,
        params: dict[str, Any],
    ) -> Any:
        """Return ``df`` unchanged, ignoring the captured snapshot."""
        return df

apply(df, params)

Return df unchanged, ignoring the captured snapshot.

Source code in skyulf-core/skyulf/preprocessing/inspection.py 
178
179
180
181
182
183
184
def apply(
    self,
    df: pd.DataFrame | SkyulfDataFrame | tuple[Any, ...] | Any,
    params: dict[str, Any],
) -> Any:
    """Return ``df`` unchanged, ignoring the captured snapshot."""
    return df

inference_capability(state, *, engine) staticmethod

Describe the existing passthrough without taking another snapshot.

Source code in skyulf-core/skyulf/preprocessing/inspection.py 
170
171
172
173
174
175
176
@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe the existing passthrough without taking another snapshot."""
    if engine not in {"pandas", "polars"}:
        return None
    DataSnapshotApplier.validate_inference_state(state)
    return ExecutionCapability(engine, "apply", "local", "preserve", "row")

validate_inference_state(raw) staticmethod

Check ownership without interpreting or recapturing training values.

Source code in skyulf-core/skyulf/preprocessing/inspection.py 
164
165
166
167
168
@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Check ownership without interpreting or recapturing training values."""
    local_state_fields(raw, "data_snapshot", {"type", "snapshot"})
    return raw

DataSnapshotCalculator

Bases: BaseCalculator

Capture the leading rows of the input as plain record dicts.

Source code in skyulf-core/skyulf/preprocessing/inspection.py 
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
@NodeRegistry.register("DataSnapshot", DataSnapshotApplier)
@node_meta(
    id="DataSnapshot",
    name="Data Snapshot",
    category="Inspection",
    description="Take a snapshot of the first N rows of the dataset.",
    params={"n_rows": 5},
    learns_from_data=False,
)
class DataSnapshotCalculator(BaseCalculator):
    """Capture the leading rows of the input as plain record dicts."""

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Return ``input_schema`` untouched."""
        # Inspection nodes are read-only; schema is unchanged.
        return input_schema

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> DataSnapshotArtifact:  # pylint: disable=arguments-differ
        """Return the first ``config["n_rows"]`` rows (default 5) as dicts.

        Rows are serialised eagerly at fit time, so the snapshot is unaffected
        by later mutation of the source frame.
        """
        return cast(
            DataSnapshotArtifact,
            fit_dual_engine(
                X, config, {"polars": _snapshot_fit_polars, "pandas": _snapshot_fit_pandas}
            ),
        )

fit(X, _y, config)

Return the first config["n_rows"] rows (default 5) as dicts.

Rows are serialised eagerly at fit time, so the snapshot is unaffected by later mutation of the source frame.

Source code in skyulf-core/skyulf/preprocessing/inspection.py 
206
207
208
209
210
211
212
213
214
215
216
217
218
@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> DataSnapshotArtifact:  # pylint: disable=arguments-differ
    """Return the first ``config["n_rows"]`` rows (default 5) as dicts.

    Rows are serialised eagerly at fit time, so the snapshot is unaffected
    by later mutation of the source frame.
    """
    return cast(
        DataSnapshotArtifact,
        fit_dual_engine(
            X, config, {"polars": _snapshot_fit_polars, "pandas": _snapshot_fit_pandas}
        ),
    )

infer_output_schema(input_schema, config)

Return input_schema untouched.

Source code in skyulf-core/skyulf/preprocessing/inspection.py 
199
200
201
202
203
204
def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Return ``input_schema`` untouched."""
    # Inspection nodes are read-only; schema is unchanged.
    return input_schema

DatasetProfileApplier

Bases: BaseApplier

Passthrough applier; the profile is produced entirely at fit time.

Source code in skyulf-core/skyulf/preprocessing/inspection.py 
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
class DatasetProfileApplier(BaseApplier):
    """Passthrough applier; the profile is produced entirely at fit time."""

    @staticmethod
    def validate_inference_state(raw: dict) -> dict:
        """Check ownership while leaving the unused training report untouched."""
        local_state_fields(raw, "dataset_profile", {"type", "profile"})
        return raw

    @staticmethod
    def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
        """Describe the existing passthrough without collecting new statistics."""
        if engine not in {"pandas", "polars"}:
            return None
        DatasetProfileApplier.validate_inference_state(state)
        return ExecutionCapability(engine, "apply", "local", "preserve", "row")

    def apply(
        self,
        df: pd.DataFrame | SkyulfDataFrame | tuple[Any, ...] | Any,
        params: dict[str, Any],
    ) -> Any:
        """Return ``df`` unchanged, ignoring the fitted profile."""
        # Inspection nodes do not modify data.
        return df

apply(df, params)

Return df unchanged, ignoring the fitted profile.

Source code in skyulf-core/skyulf/preprocessing/inspection.py 
 95
 96
 97
 98
 99
100
101
102
def apply(
    self,
    df: pd.DataFrame | SkyulfDataFrame | tuple[Any, ...] | Any,
    params: dict[str, Any],
) -> Any:
    """Return ``df`` unchanged, ignoring the fitted profile."""
    # Inspection nodes do not modify data.
    return df

inference_capability(state, *, engine) staticmethod

Describe the existing passthrough without collecting new statistics.

Source code in skyulf-core/skyulf/preprocessing/inspection.py 
87
88
89
90
91
92
93
@staticmethod
def inference_capability(state: dict, *, engine: str) -> ExecutionCapability | None:
    """Describe the existing passthrough without collecting new statistics."""
    if engine not in {"pandas", "polars"}:
        return None
    DatasetProfileApplier.validate_inference_state(state)
    return ExecutionCapability(engine, "apply", "local", "preserve", "row")

validate_inference_state(raw) staticmethod

Check ownership while leaving the unused training report untouched.

Source code in skyulf-core/skyulf/preprocessing/inspection.py 
81
82
83
84
85
@staticmethod
def validate_inference_state(raw: dict) -> dict:
    """Check ownership while leaving the unused training report untouched."""
    local_state_fields(raw, "dataset_profile", {"type", "profile"})
    return raw

DatasetProfileCalculator

Bases: BaseCalculator

Summarise shape, dtypes, missingness and numeric statistics.

Numeric statistics include supported integer, unsigned integer, float and Decimal columns, even when binary, constant, entirely missing or empty. Boolean, categorical and temporal columns retain their dtype/missingness metadata but are not treated as numeric features.

Source code in skyulf-core/skyulf/preprocessing/inspection.py 
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
@NodeRegistry.register("DatasetProfile", DatasetProfileApplier)
@node_meta(
    id="DatasetProfile",
    name="Dataset Profile",
    category="Inspection",
    description="Generate a statistical profile of the dataset.",
    params={},
    learns_from_data=False,
)
class DatasetProfileCalculator(BaseCalculator):
    """Summarise shape, dtypes, missingness and numeric statistics.

    Numeric statistics include supported integer, unsigned integer, float and
    Decimal columns, even when binary, constant, entirely missing or empty.
    Boolean, categorical and temporal columns retain their dtype/missingness
    metadata but are not treated as numeric features.
    """

    def infer_output_schema(
        self, input_schema: SkyulfSchema, config: dict[str, Any]
    ) -> SkyulfSchema:
        """Return ``input_schema`` untouched."""
        # Inspection nodes are read-only; schema is unchanged.
        return input_schema

    @fit_method
    def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> DatasetProfileArtifact:  # pylint: disable=arguments-differ
        """Collect row/column counts, dtypes, missing counts and numeric stats.

        Numeric statistics come from Polars ``describe()`` or pandas
        ``describe()`` depending on the engine, so the per-column metric names
        are not identical across engines.
        """
        return cast(
            DatasetProfileArtifact,
            fit_dual_engine(
                X, config, {"polars": _profile_fit_polars, "pandas": _profile_fit_pandas}
            ),
        )

fit(X, _y, config)

Collect row/column counts, dtypes, missing counts and numeric stats.

Numeric statistics come from Polars describe() or pandas describe() depending on the engine, so the per-column metric names are not identical across engines.

Source code in skyulf-core/skyulf/preprocessing/inspection.py 
130
131
132
133
134
135
136
137
138
139
140
141
142
143
@fit_method
def fit(self, X: Any, _y: Any, config: dict[str, Any]) -> DatasetProfileArtifact:  # pylint: disable=arguments-differ
    """Collect row/column counts, dtypes, missing counts and numeric stats.

    Numeric statistics come from Polars ``describe()`` or pandas
    ``describe()`` depending on the engine, so the per-column metric names
    are not identical across engines.
    """
    return cast(
        DatasetProfileArtifact,
        fit_dual_engine(
            X, config, {"polars": _profile_fit_polars, "pandas": _profile_fit_pandas}
        ),
    )

infer_output_schema(input_schema, config)

Return input_schema untouched.

Source code in skyulf-core/skyulf/preprocessing/inspection.py 
123
124
125
126
127
128
def infer_output_schema(
    self, input_schema: SkyulfSchema, config: dict[str, Any]
) -> SkyulfSchema:
    """Return ``input_schema`` untouched."""
    # Inspection nodes are read-only; schema is unchanged.
    return input_schema