File Coverage

blib/lib/Algorithm/Classifier/IsolationForest/Online.pm
Criterion Covered Total %
statement 482 487 98.9
branch 177 206 85.9
condition 84 124 67.7
subroutine 72 73 98.6
pod 28 28 100.0
total 843 918 91.8


line stmt bran cond sub pod time code
1             package Algorithm::Classifier::IsolationForest::Online;
2              
3 90     90   200049 use strict;
  90         152  
  90         3668  
4 90     90   1125 use warnings;
  90         355  
  90         4213  
5 90     90   486 use Carp qw(croak);
  90         238  
  90         4213  
6 90     90   1019 use JSON::PP ();
  90         15225  
  90         2016  
7 90     90   1188 use File::Slurp qw(read_file write_file);
  90         38203  
  90         4079  
8              
9             # Runtime-only dependency: tagged_row_to_array is delegated to the parent
10             # class (identical semantics, no point duplicating it) and the
11             # contamination threshold selection reuses _threshold_from_ranked. The
12             # parent never loads this module at compile time (its from_json requires
13             # it on demand), so there is no cycle.
14 90     90   2571 use Algorithm::Classifier::IsolationForest ();
  90         124  
  90         3352  
15              
16             our $VERSION = '0.6.0';
17              
18             # Node layout. Unlike the batch forest's nodes, online nodes are mutable
19             # and carry a running point count plus the bounding box (per-feature
20             # lo/hi) of every point that has passed through them -- that box is what
21             # split simulation samples from, since points themselves are never stored
22             # in the tree. Both node types share the first four slots so the
23             # learn/unlearn bookkeeping never has to branch on type:
24             #
25             # leaf: [0, count, \@lo, \@hi]
26             # internal: [1, count, \@lo, \@hi, attr, split, left, right]
27             #
28             # The type tag mirrors the parent's convention (0 is falsy, so
29             # while ($node->[0]) walks to a leaf). A leaf built from an empty
30             # synthetic partition has count 0 and an undef box (slots 2/3); the box
31             # is initialised from the first real point that reaches it.
32 90     90   347 use constant _N_TYPE => 0;
  90         130  
  90         6173  
33 90     90   404 use constant _N_COUNT => 1;
  90         198  
  90         4249  
34 90     90   368 use constant _N_LO => 2;
  90         164  
  90         3239  
35 90     90   367 use constant _N_HI => 3;
  90         220  
  90         3029  
36 90     90   350 use constant _N_ATTR => 4;
  90         147  
  90         3250  
37 90     90   391 use constant _N_SPLIT => 5;
  90         164  
  90         2912  
38 90     90   372 use constant _N_LEFT => 6;
  90         132  
  90         2759  
39 90     90   328 use constant _N_RIGHT => 7;
  90         150  
  90         2775  
40              
41 90     90   340 use constant _NT_LEAF => 0;
  90         129  
  90         2587  
42 90     90   370 use constant _NT_AXIS => 1;
  90         165  
  90         4759  
43              
44             # Trees are binary (the reference implementation's branching_factor == 2),
45             # which fixes the depth-budget log base at log(2 * 2). Spelled as the
46             # exact-double literal rather than log(4) so it is bit-identical to the
47             # OL_LOG4 literal the C learn path uses regardless of the platform's
48             # libm rounding -- a one-ulp disagreement would flip `depth < limit`
49             # split decisions exactly when a tree's count is eta * 4**k (the same
50             # TWO_PI trick the parent uses for _randn parity).
51 90     90   453 use constant _LOG4 => unpack( 'd', pack 'd', 1.3862943611198906 );
  90         346  
  90         4064  
52 90     90   351 use constant _LOG2 => log(2);
  90         145  
  90         4040  
53              
54             # DBL_EPSILON, added to the normalisation factor before dividing so a
55             # just-started model (normaliser 0) yields well-defined scores instead of
56             # a division by zero -- the same guard the reference implementation uses.
57 90     90   368 use constant _EPS => 2.220446049250313e-16;
  90         129  
  90         4203  
58              
59             # The online learn/unlearn/score-row XS functions were added to the C
60             # backend after the batch-scoring ones, so a prebuilt object installed
61             # from an older release can back $HAS_C while lacking them (the parent
62             # trusts a flag-matched prebuilt object without inspecting its symbol
63             # set). Probe once at load: without them, use_c still accelerates the
64             # packed-snapshot batch scoring -- those functions have been in the
65             # object all along -- and learning quietly stays pure Perl instead of
66             # crashing on an undefined XS sub. Rebuilding/reinstalling (or
67             # IF_RUNTIME_BUILD=1) restores the full set.
68 90 50   90   376 use constant _HAS_ONLINE_XS => defined &Algorithm::Classifier::IsolationForest::online_learn_row_xs ? 1 : 0;
  90         142  
  90         494472  
69              
70             =head1 NAME
71              
72             Algorithm::Classifier::IsolationForest::Online - Online (streaming) Isolation Forest anomaly detection
73              
74             =head1 SYNOPSIS
75              
76             use Algorithm::Classifier::IsolationForest::Online;
77              
78             my $oif = Algorithm::Classifier::IsolationForest::Online->new(
79             n_trees => 100,
80             window_size => 2048,
81             max_leaf_samples => 32,
82             seed => 42,
83             );
84              
85             # stream data through the model; each point is learned and old
86             # points beyond the window are forgotten automatically
87             $oif->learn(\@warmup_rows);
88              
89             # prequential operation: score each point against the model as it
90             # stood BEFORE that point was learned, then learn it
91             my $scores = $oif->score_learn(\@new_rows);
92              
93             # or score without learning
94             my $scores2 = $oif->score_samples(\@query_rows);
95             my $labels = $oif->predict(\@query_rows);
96              
97             # persistence keeps the window, so a reloaded model keeps forgetting
98             # correctly as the stream continues
99             $oif->save('oiforest_model.json');
100             my $resumed = Algorithm::Classifier::IsolationForest::Online->load('oiforest_model.json');
101              
102             =head1 DESCRIPTION
103              
104             Implements Online Isolation Forest (Online-iForest; Leveni, Weigert
105             Cassales, Pfahringer, Bifet & Boracchi 2024 -- see REFERENCES), a
106             streaming variant of Isolation Forest for data that arrives continuously
107             and whose distribution may drift. There is no C: the model
108             Cs points as they arrive and, once more than C points
109             have been seen, forgets the oldest point for every new one so the model
110             always reflects the most recent C points of the stream.
111              
112             Trees never store data points. Each node keeps only a running count of
113             the points that passed through it and the bounding box of their feature
114             values. A leaf splits once enough points have accumulated (see
115             C and C); because the actual points are gone,
116             the split simulates them by sampling uniformly inside the leaf's bounding
117             box. Forgetting reverses the process: counts are decremented along the
118             forgotten point's path and a subtree whose count falls below its split
119             requirement is collapsed back into a leaf.
120              
121             Scoring follows the classic Isolation Forest intuition -- anomalies
122             isolate at shallow depth -- but normalises by the depth budget
123             C of the current window rather than the
124             batch model's C. Scores are in (0, 1] with high values
125             anomalous, directly comparable in spirit (though not numerically) to the
126             parent class's scores.
127              
128             Both learning and scoring are accelerated through the parent class's
129             Inline::C backend when it is available; C covers them together.
130              
131             Learning (and the per-row walks inside C) runs in C
132             directly against the live trees, drawing randomness through the same
133             generator in the same order as the pure-Perl path -- so, like the
134             parent's C, a C with a given seed produces bit-identical
135             trees whether C is on or off (on C perls; wide-NV
136             perls keep extra low bits in the pure-Perl path). The knob changes
137             speed, never results.
138              
139             Batch scoring lazily flattens the mutable trees into the same packed
140             node layout the batch scorer walks -- online trees are axis-only, and
141             the online per-leaf depth adjustment rides in the slot the batch packer
142             uses for its own leaf adjustment -- so C, C,
143             C, C, and C
144             all run through the same C (and OpenMP, when linked) tree walk the
145             parent uses, with identical results to the pure-Perl fallback. Any
146             C invalidates the packed snapshot; the next batch-scoring call
147             repacks once. C never touches the snapshot: it mutates
148             the trees after every single point, so its rows are scored by walking
149             the live trees in C instead.
150              
151             A model needs to have seen at least C points before
152             tree structure exists at all; until then every point scores 1.0. Give
153             the model a warm-up C pass before trusting scores or labels.
154              
155             Models saved by this class carry their own C tag.
156             C<< Algorithm::Classifier::IsolationForest->load >> recognises it and
157             dispatches here, so callers can load either model type through the
158             parent class.
159              
160             =head1 GENERAL METHODS
161              
162             =head2 new(%args)
163              
164             Inits the object.
165              
166             - n_trees :: number of isolation trees in the ensemble
167             default :: 100
168              
169             - window_size :: how many of the most recent points the model reflects.
170             Once the stream exceeds this, learning a point forgets the
171             oldest retained point. 0 or undef disables forgetting: the
172             model then learns from the whole stream and retains no window
173             (so nothing is ever unlearned and threshold relearning needs
174             caller-supplied data).
175             default :: 2048
176              
177             - max_leaf_samples :: how many points a leaf must accumulate before it
178             splits (eta in the paper). Also the unit of the depth budget:
179             trees stop splitting past log(n/eta)/log(4).
180             default :: 32
181              
182             - growth :: how the split requirement scales with depth (the reference
183             implementation's `type` parameter).
184             adaptive :: a leaf at depth k needs max_leaf_samples * 2**k
185             points to split -- deeper splits need
186             exponentially more evidence
187             fixed :: max_leaf_samples points regardless of depth
188             default :: adaptive
189              
190             - subsample :: probability in (0, 1] that a given tree learns (or
191             forgets) a given point, drawn independently per tree per point.
192             Values below 1 increase diversity among trees on very dense
193             streams. Note that, as in the reference implementation, learn
194             and forget draws are independent, so per-tree counts are only
195             approximate under subsampling.
196             default :: 1.0
197              
198             - seed :: optional integer to seed srand with, for reproducible trees
199             given the same stream in the same order. Processed via
200             abs(int()). Seeding happens here in new(), since there is no
201             fit() to do it in.
202             default :: undef
203              
204             - contamination :: expected fraction of anomalies, in (0, 0.5]. When
205             set, the first predict()-family call learns a score threshold
206             that flags this fraction of the current window, and uses it as
207             the default cutoff. The threshold does NOT track the stream
208             automatically afterwards; call relearn_threshold() to refresh
209             it. undef => no learned threshold (predict() falls back to
210             0.5).
211             default :: undef
212              
213             - missing :: how learn() treats undef (missing) feature cells. Scoring
214             always tolerates undef (mapped to 0), matching the parent
215             class's long-standing behaviour.
216             die :: croak if a learned point contains an undef cell
217             zero :: treat a missing cell as the value 0
218             default :: die
219              
220             - feature_names :: optional arrayref of per-feature labels enabling the
221             *_tagged methods (and required by mungers below).
222             default :: undef
223              
224             - mungers :: optional hashref of declarative L
225             specs; every tagged row (learn_tagged, score_learn_tagged, the
226             scoring *_tagged methods, tagged_row_to_array) is munged from
227             raw values into numbers through the compiled plan, and
228             munge_rows() applies the scalar mungers to positional rows.
229             Requires feature_names; spec errors croak here. The spec is
230             saved with the model. Identical semantics to the parent
231             class's knob -- see MUNGERS in
232             L for details and
233             caveats.
234             default :: undef
235              
236             - schema_version :: optional opaque string identifying the revision of
237             the variable schema this model was built against. Never
238             parsed; saved with the model. Usually set via a prototype --
239             see PROTOTYPES in L
240             (whose new_from_prototype creates online models too).
241             default :: undef
242              
243             - schema_description :: optional opaque free-text description of what
244             the variable schema is. Same handling as schema_version.
245             default :: undef
246              
247             - feature_descriptions :: optional hashref of 'feature name => free
248             text' describing individual features. Requires feature_names;
249             every key must name an entry there or new() croaks. Partial
250             coverage is fine. Saved with the model.
251             default :: undef
252              
253             - use_c :: boolean, override whether the parent class's Inline::C
254             backend is used, for learning and scoring both (see
255             DESCRIPTION). When false the instance runs pure Perl even if
256             the C backend compiled. Results are identical either way --
257             learn() builds bit-identical trees for the same seed (on
258             nvsize == 8 perls) and scoring matches exactly -- so only
259             speed differs.
260             default :: $Algorithm::Classifier::IsolationForest::HAS_C
261              
262             - use_openmp :: boolean, override whether OpenMP parallel scoring is
263             used inside the C tree walk. Ignored when use_c is false.
264             default :: $Algorithm::Classifier::IsolationForest::HAS_OPENMP
265              
266             =cut
267              
268             sub new {
269 73     73 1 753566 my ( $class, %args ) = @_;
270              
271 73   100     424 my $growth = $args{growth} // 'adaptive';
272 73 100       713 croak "growth must be 'adaptive' or 'fixed'"
273             unless $growth =~ /\A(?:adaptive|fixed)\z/;
274              
275 72   100     281 my $missing = $args{missing} // 'die';
276 72 100       413 croak "missing must be one of: die, zero"
277             unless $missing =~ /\A(?:die|zero)\z/;
278              
279 71 100       275 if ( defined( $args{seed} ) ) {
280 44         125 $args{seed} = abs( int( $args{seed} ) );
281             }
282              
283             # window_size => 0 and window_size => undef both mean "no forgetting";
284             # normalise to 0 so the rest of the code has one falsy spelling.
285 71 100 100     304 my $window_size = exists $args{window_size} ? ( $args{window_size} // 0 ) : 2048;
286              
287             # Clamp the accel knobs against what the parent's build actually has,
288             # exactly as the parent's new() does: use_c => 1 without a compiled
289             # backend would otherwise call undefined XS subs at first scoring, and
290             # OpenMP is meaningless without the C tree walk.
291             my $use_c
292             = defined $args{use_c}
293 71 100 66     295 ? ( $args{use_c} && $Algorithm::Classifier::IsolationForest::HAS_C ? 1 : 0 )
    100          
294             : $Algorithm::Classifier::IsolationForest::HAS_C;
295             my $use_openmp
296             = defined $args{use_openmp}
297 71 50 33     185 ? ( $args{use_openmp} && $Algorithm::Classifier::IsolationForest::HAS_OPENMP ? 1 : 0 )
    100          
298             : $Algorithm::Classifier::IsolationForest::HAS_OPENMP;
299 71 100       220 $use_openmp = 0 unless $use_c;
300              
301             my $self = {
302             n_trees => $args{n_trees} // 100,
303             window_size => $window_size,
304             max_leaf_samples => $args{max_leaf_samples} // 32,
305             growth => $growth,
306             subsample => $args{subsample} // 1.0,
307             seed => $args{seed},
308             contamination => $args{contamination},
309             missing => $missing,
310             feature_names => $args{feature_names},
311             threshold => undef, # learned lazily if contamination set
312             n_features => undef, # learned from the first row
313             seen => 0, # total points learned over the model's lifetime
314             window => [], # the retained rows, oldest first
315             trees => [],
316             mungers => undef, # optional Algorithm::ToNumberMunger spec hash
317             # Opaque schema metadata, usually set via the parent class's
318             # new_from_prototype and persisted with the model.
319             schema_version => $args{schema_version},
320             schema_description => $args{schema_description},
321             feature_descriptions => $args{feature_descriptions},
322 71   100     1245 _use_c => $use_c,
      100        
      100        
323             _use_openmp => $use_openmp,
324             };
325              
326 71         183 for my $doc (qw(schema_version schema_description)) {
327             croak "$doc must be a plain string"
328 142 50 66     426 if defined $self->{$doc} && ref $self->{$doc};
329             }
330             Algorithm::Classifier::IsolationForest::_validate_feature_descriptions( $self->{feature_names},
331             $self->{feature_descriptions} )
332 71 100       210 if defined $self->{feature_descriptions};
333              
334             # Optional Algorithm::ToNumberMunger integration, identical to the
335             # parent's: compiled eagerly so spec errors surface here; the module
336             # is only required when a spec is actually given.
337 70 100       175 if ( defined $args{mungers} ) {
338             croak "mungers must be a hashref of 'tag => munger spec'"
339 4 50       14 unless ref $args{mungers} eq 'HASH';
340             croak "mungers requires feature_names (the munger plan compiles against them)"
341 4 100 66     145 unless ref $self->{feature_names} eq 'ARRAY' && @{ $self->{feature_names} };
  3         10  
342 3         11 $self->{mungers} = $args{mungers};
343             $self->{_munger_plan}
344 3         20 = Algorithm::Classifier::IsolationForest::_compile_mungers( $self->{feature_names}, $self->{mungers} );
345 3         466 $self->{munger_module_version} = $Algorithm::ToNumberMunger::VERSION;
346             } ## end if ( defined $args{mungers} )
347              
348 69 100       271 croak "n_trees must be >= 1" unless $self->{n_trees} >= 1;
349 68 100       269 croak "max_leaf_samples must be >= 1" unless $self->{max_leaf_samples} >= 1;
350             croak "window_size must be 0 (unbounded) or >= max_leaf_samples"
351 67 100 100     401 if $self->{window_size} && $self->{window_size} < $self->{max_leaf_samples};
352             croak "subsample must be in (0, 1]"
353 66 100 100     693 unless $self->{subsample} > 0 && $self->{subsample} <= 1;
354             croak "contamination must be a number in (0, 0.5]"
355             if defined $self->{contamination}
356 63 100 100     518 && !( $self->{contamination} > 0 && $self->{contamination} <= 0.5 );
      100        
357              
358 61         265 $self->{trees} = [ map { { root => undef, count => 0, depth_limit => 0 } } 1 .. $self->{n_trees} ];
  2000         4092  
359              
360 61 100       302 srand( $self->{seed} ) if defined $self->{seed};
361              
362 61         362 return bless $self, $class;
363             } ## end sub new
364              
365             =head2 learn(\@data)
366              
367             Learns the passed samples, in order, as the next points of the stream.
368             Once the model has seen more than C points, each learned
369             point also forgets the oldest retained point, so the model tracks the
370             most recent C points.
371              
372             The data format matches the parent class's C: an arrayref of
373             arrayrefs, each inner arrayref one sample of numeric features. All
374             samples must have the same feature count; the count is locked in by the
375             first sample ever learned.
376              
377             Returns C<$self>, so it chains.
378              
379             $oif->learn(\@rows);
380              
381             =cut
382              
383             sub learn {
384 262     262 1 107707 my ( $self, $data ) = @_;
385 262 50 33     799 croak "learn() expects a non-empty arrayref of samples"
386             unless ref $data eq 'ARRAY' && @$data;
387 262         406 for my $row (@$data) {
388 14493         25220 $self->_learn_row( $self->_prep_row( $row, 'learn' ) );
389             }
390 260         586 return $self;
391             }
392              
393             =head2 learn_tagged(\%row)
394              
395             =head2 learn_tagged(\@rows)
396              
397             Learns one sample supplied as a hashref of named feature values, or a
398             whole batch supplied as an arrayref of such hashrefs, in stream order.
399             The model must have C set. Rows go through
400             L (and therefore through the munger plan when
401             C is configured). Returns C<$self>.
402              
403             $oif->learn_tagged({ cpu => 0.9, mem => 0.4, disk => 0.1 });
404             $oif->learn_tagged(\@hashref_rows);
405              
406             Croaks under the same conditions as L, naming the
407             offending row by index in the batch form.
408              
409             =cut
410              
411             sub learn_tagged {
412 3     3 1 442 my ( $self, $row ) = @_;
413 3 100       13 if ( ref $row eq 'ARRAY' ) {
414 1         2 my @rows;
415 1         4 for my $i ( 0 .. $#$row ) {
416 200         360 push @rows, $self->tagged_row_to_array( $row->[$i], "learn_tagged (row $i)" );
417             }
418 1         6 return $self->learn( \@rows );
419             }
420 2         10 my $vec = $self->tagged_row_to_array( $row, 'learn_tagged' );
421 2         8 return $self->learn( [$vec] );
422             } ## end sub learn_tagged
423              
424             =head2 score_learn(\@data)
425              
426             Prequential (test-then-train) operation, the usual way to run a streaming
427             detector: each sample is scored against the model as it stood I
428             that sample was learned, then learned. Returns an arrayref of anomaly
429             scores, one per sample, in input order.
430              
431             Unlike the pure scoring methods this works on a brand-new model too (the
432             first points of a stream simply score 1.0, as nothing is known yet).
433              
434             my $scores = $oif->score_learn(\@rows);
435              
436             =cut
437              
438             sub score_learn {
439 35     35 1 8599 my ( $self, $data ) = @_;
440 35 50 33     233 croak "score_learn() expects a non-empty arrayref of samples"
441             unless ref $data eq 'ARRAY' && @$data;
442 35         70 my @scores;
443 35         91 for my $row (@$data) {
444 1925         3528 my $r = $self->_prep_row( $row, 'score_learn' );
445 1924         3577 push @scores, $self->_score_row($r);
446 1924         3087 $self->_learn_row($r);
447             }
448 34         485 return \@scores;
449             } ## end sub score_learn
450              
451             =head2 score_learn_tagged(\%row)
452              
453             Prequential score-then-learn for a single sample supplied as a hashref of
454             named feature values. Returns the scalar anomaly score the sample had
455             before it was learned.
456              
457             my $score = $oif->score_learn_tagged({ cpu => 0.9, mem => 0.4 });
458              
459             Croaks under the same conditions as L.
460              
461             =cut
462              
463             sub score_learn_tagged {
464 2     2 1 1580 my ( $self, $row ) = @_;
465 2         14 my $vec = $self->tagged_row_to_array( $row, 'score_learn_tagged' );
466 2         15 my $result = $self->score_learn( [$vec] );
467 2         16 return $result->[0];
468             }
469              
470             =head2 score_samples(\@data)
471              
472             Returns an arrayref of anomaly scores in (0, 1] without learning
473             anything. Scores near 1 are strong anomalies (isolated at shallow
474             depth); scores well below 0.5 are normal.
475              
476             my $scores = $oif->score_samples(\@data);
477              
478             =cut
479              
480             sub score_samples {
481 69     69 1 51885 my ( $self, $data ) = @_;
482 69         292 $self->_check_learned;
483 68 50       203 croak "score_samples() expects an arrayref of samples"
484             unless ref $data eq 'ARRAY';
485              
486 68 100       226 if ( $self->_ensure_c_trees ) {
487 55         188 my ( $n_pts, $x_packed ) = $self->_pack_input($data);
488 55         202 my $sums_packed = "\0" x ( $n_pts * 8 );
489             Algorithm::Classifier::IsolationForest::score_all_xs(
490             $self->{_c_nodes}, $self->{_c_coef_idx}, $self->{_c_coef_val},
491             $x_packed, $sums_packed, $n_pts,
492 55         256518 $self->{n_features}, scalar @{ $self->{trees} }, $self->{_use_openmp}
493 55         186 );
494 55         311 my $result = [];
495 55         339 Algorithm::Classifier::IsolationForest::finalize_scores_xs( $sums_packed, $n_pts, $self->_score_inv, $result );
496 55         2824 return $result;
497             } ## end if ( $self->_ensure_c_trees )
498              
499 13         33 my $sums = $self->_depth_sums($data);
500 13         31 my $inv = $self->_score_inv;
501 13         32 return [ map { exp( -$_ * $inv ) } @$sums ];
  567         779  
502             } ## end sub score_samples
503              
504             =head2 score_sample_tagged(\%row)
505              
506             Scores a single sample supplied as a hashref of named feature values,
507             without learning it. Returns a scalar anomaly score in (0, 1].
508              
509             my $score = $oif->score_sample_tagged({ cpu => 0.9, mem => 0.4 });
510              
511             Croaks under the same conditions as L.
512              
513             =cut
514              
515             sub score_sample_tagged {
516 7     7 1 2171 my ( $self, $row ) = @_;
517 7         24 my $vec = $self->tagged_row_to_array( $row, 'score_sample_tagged' );
518 4         19 my $result = $self->score_samples( [$vec] );
519 4         39 return $result->[0];
520             }
521              
522             =head2 path_lengths(\@data)
523              
524             Returns an arrayref of the mean isolation depth per sample across the
525             trees, for inspection -- the streaming counterpart of the parent class's
526             method of the same name. Depths include the per-leaf count adjustment.
527              
528             my $depths = $oif->path_lengths(\@data);
529              
530             =cut
531              
532             sub path_lengths {
533 4     4 1 3608 my ( $self, $data ) = @_;
534 4         14 $self->_check_learned;
535 3 50       10 croak "path_lengths() expects an arrayref of samples"
536             unless ref $data eq 'ARRAY';
537 3         6 my $t = scalar @{ $self->{trees} };
  3         6  
538              
539 3 100       71 if ( $self->_ensure_c_trees ) {
540 2         6 my ( $n_pts, $x_packed ) = $self->_pack_input($data);
541 2         7 my $sums_packed = "\0" x ( $n_pts * 8 );
542             Algorithm::Classifier::IsolationForest::score_all_xs(
543             $self->{_c_nodes}, $self->{_c_coef_idx}, $self->{_c_coef_val},
544             $x_packed, $sums_packed, $n_pts,
545             $self->{n_features}, $t, $self->{_use_openmp}
546 2         2915 );
547 2         7 my $result = [];
548 2         14 Algorithm::Classifier::IsolationForest::finalize_path_lengths_xs( $sums_packed, $n_pts, $t + 0.0, $result );
549 2         11 return $result;
550             } ## end if ( $self->_ensure_c_trees )
551              
552 1         3 my $sums = $self->_depth_sums($data);
553 1         3 return [ map { $_ / $t } @$sums ];
  5         12  
554             } ## end sub path_lengths
555              
556             =head2 predict(\@data, $threshold)
557              
558             Returns an arrayref of 0/1 labels for the specified data, without
559             learning it.
560              
561             If C<$threshold> is not given, the contamination-learned cutoff is used
562             when available (learned from the current window on first use -- see
563             C in L), otherwise 0.5.
564              
565             Note that absolute score levels depend on C and
566             C (shallower depth budgets compress scores downward),
567             so the 0.5 fallback is a blunt default here -- anomalies reliably rank
568             above normal points, but may sit below 0.5. Setting C,
569             or passing a threshold calibrated from observed scores, is recommended.
570              
571             my $labels = $oif->predict(\@data);
572              
573             =cut
574              
575             sub predict {
576 17     17 1 2314 my ( $self, $data, $threshold ) = @_;
577 17         60 $self->_check_learned;
578 16         106 $self->_ensure_threshold;
579             $threshold
580             = defined $threshold ? $threshold
581             : defined $self->{threshold} ? $self->{threshold}
582 16 50       59 : 0.5;
    100          
583              
584             # Fast path: threshold the raw depth sums directly, skipping the
585             # per-point exp() -- score >= T iff sum <= -log(T)/inv. Only valid
586             # for a normal threshold in (0, 1), like the parent's gate.
587 16 100 66     91 if ( $threshold > 0 && $threshold < 1 && $self->_ensure_c_trees ) {
      100        
588 9         27 my ( $n_pts, $x_packed ) = $self->_pack_input($data);
589 9         33 my $sums_packed = "\0" x ( $n_pts * 8 );
590             Algorithm::Classifier::IsolationForest::score_all_xs(
591             $self->{_c_nodes}, $self->{_c_coef_idx}, $self->{_c_coef_val},
592             $x_packed, $sums_packed, $n_pts,
593 9         25770 $self->{n_features}, scalar @{ $self->{trees} }, $self->{_use_openmp}
594 9         28 );
595 9         77 my $sum_threshold = -log($threshold) / $self->_score_inv;
596 9         22 my $result = [];
597 9         74 Algorithm::Classifier::IsolationForest::predict_sums_xs( $sums_packed, $n_pts, $sum_threshold, $result );
598 9         45 return $result;
599             } ## end if ( $threshold > 0 && $threshold < 1 && $self...)
600              
601 7         19 my $scores = $self->score_samples($data);
602 7 100       12 return [ map { $_ >= $threshold ? 1 : 0 } @$scores ];
  35         84  
603             } ## end sub predict
604              
605             =head2 predict_tagged(\%row, $threshold)
606              
607             Predicts whether a single sample, supplied as a hashref of named feature
608             values, is an anomaly. Returns a scalar 1 (anomaly) or 0 (normal).
609             C<$threshold> defaults the same way as in L.
610              
611             my $label = $oif->predict_tagged({ cpu => 0.9, mem => 0.4 });
612              
613             Croaks under the same conditions as L.
614              
615             =cut
616              
617             sub predict_tagged {
618 1     1 1 1245 my ( $self, $row, $threshold ) = @_;
619 1         8 my $vec = $self->tagged_row_to_array( $row, 'predict_tagged' );
620 1         9 my $result = $self->predict( [$vec], $threshold );
621 1         10 return $result->[0];
622             }
623              
624             =head2 score_predict_samples(\@data, $threshold)
625              
626             Returns an arrayref of C<[$score, $label]> pairs, one per sample, without
627             learning. C<$threshold> defaults the same way as in L.
628              
629             my $results = $oif->score_predict_samples(\@data);
630              
631             =cut
632              
633             sub score_predict_samples {
634 4     4 1 872 my ( $self, $data, $threshold ) = @_;
635 4         16 $self->_check_learned;
636 4         15 $self->_ensure_threshold;
637             $threshold
638             = defined $threshold ? $threshold
639             : defined $self->{threshold} ? $self->{threshold}
640 4 50       15 : 0.5;
    100          
641              
642             # Fast path: [score, label] pairs built straight from the sum buffer
643             # in one C call; gated identically to predict().
644 4 100 33     35 if ( $threshold > 0 && $threshold < 1 && $self->_ensure_c_trees ) {
      66        
645 3         12 my ( $n_pts, $x_packed ) = $self->_pack_input($data);
646 3         10 my $sums_packed = "\0" x ( $n_pts * 8 );
647             Algorithm::Classifier::IsolationForest::score_all_xs(
648             $self->{_c_nodes}, $self->{_c_coef_idx}, $self->{_c_coef_val},
649             $x_packed, $sums_packed, $n_pts,
650 3         27429 $self->{n_features}, scalar @{ $self->{trees} }, $self->{_use_openmp}
651 3         12 );
652 3         44 my $inv = $self->_score_inv;
653 3         10 my $sum_threshold = -log($threshold) / $inv;
654 3         8 my $result = [];
655 3         72 Algorithm::Classifier::IsolationForest::score_predict_xs( $sums_packed, $n_pts, $inv, $sum_threshold, $result );
656 3         21 return $result;
657             } ## end if ( $threshold > 0 && $threshold < 1 && $self...)
658              
659 1         3 my $scores = $self->score_samples($data);
660 1 100       4 return [ map { [ $_, ( $_ >= $threshold ? 1 : 0 ) ] } @$scores ];
  5         24  
661             } ## end sub score_predict_samples
662              
663             =head2 score_predict_sample_tagged(\%row, $threshold)
664              
665             Scores and classifies a single sample supplied as a hashref of named
666             feature values. Returns a two-element arrayref C<[$score, $label]>.
667             C<$threshold> defaults the same way as in L.
668              
669             my $pair = $oif->score_predict_sample_tagged({ cpu => 0.9, mem => 0.4 });
670              
671             Croaks under the same conditions as L.
672              
673             =cut
674              
675             sub score_predict_sample_tagged {
676 1     1 1 1737 my ( $self, $row, $threshold ) = @_;
677 1         9 my $vec = $self->tagged_row_to_array( $row, 'score_predict_sample_tagged' );
678 1         11 my $result = $self->score_predict_samples( [$vec], $threshold );
679 1         25 return $result->[0];
680             }
681              
682             =head2 score_predict_split(\@data, $threshold)
683              
684             Same values as L but returned as two flat
685             arrayrefs. In list context returns C<($scores_aref, $labels_aref)>.
686              
687             my ($scores, $labels) = $oif->score_predict_split(\@data);
688              
689             =cut
690              
691             sub score_predict_split {
692 3     3 1 1053 my ( $self, $data, $threshold ) = @_;
693 3         11 $self->_check_learned;
694 3         10 $self->_ensure_threshold;
695             $threshold
696             = defined $threshold ? $threshold
697             : defined $self->{threshold} ? $self->{threshold}
698 3 50       11 : 0.5;
    100          
699              
700             # Fast path: two flat arrayrefs straight from the sum buffer; gated
701             # identically to predict().
702 3 100 33     27 if ( $threshold > 0 && $threshold < 1 && $self->_ensure_c_trees ) {
      66        
703 2         9 my ( $n_pts, $x_packed ) = $self->_pack_input($data);
704 2         8 my $sums_packed = "\0" x ( $n_pts * 8 );
705             Algorithm::Classifier::IsolationForest::score_all_xs(
706             $self->{_c_nodes}, $self->{_c_coef_idx}, $self->{_c_coef_val},
707             $x_packed, $sums_packed, $n_pts,
708 2         5305 $self->{n_features}, scalar @{ $self->{trees} }, $self->{_use_openmp}
709 2         9 );
710 2         13 my $inv = $self->_score_inv;
711 2         6 my $sum_threshold = -log($threshold) / $inv;
712 2         20 my $scores = [];
713 2         5 my $labels = [];
714 2         14 Algorithm::Classifier::IsolationForest::score_predict_split_xs( $sums_packed, $n_pts, $inv,
715             $sum_threshold, $scores, $labels );
716 2         13 return ( $scores, $labels );
717             } ## end if ( $threshold > 0 && $threshold < 1 && $self...)
718              
719 1         2 my $scores = $self->score_samples($data);
720 1 100       2 my @labels = map { $_ >= $threshold ? 1 : 0 } @$scores;
  5         9  
721 1         4 return ( $scores, \@labels );
722             } ## end sub score_predict_split
723              
724             =head2 relearn_threshold(\@data)
725              
726             Re-derives the contamination decision threshold so it flags the requested
727             fraction of the current window (or of C<\@data>, when passed). Call this
728             after the stream has drifted, or on whatever cadence threshold freshness
729             matters; learning alone never moves the threshold.
730              
731             Requires C to have been set. With C<< window_size => 0 >>
732             no window is retained, so C<\@data> must be supplied.
733              
734             Returns C<$self>, so it chains.
735              
736             $oif->relearn_threshold;
737              
738             =cut
739              
740             sub relearn_threshold {
741 9     9 1 99 my ( $self, $data ) = @_;
742             croak "relearn_threshold requires contamination to have been set in new()"
743 9 100       454 unless defined $self->{contamination};
744 7 100       26 my $rows = defined $data ? $data : $self->{window};
745 7 100 66     210 croak "relearn_threshold: no retained window to learn a threshold from "
746             . "(window_size is 0); pass an arrayref of recent data"
747             unless ref $rows eq 'ARRAY' && @$rows;
748              
749 6         29 my $scores = $self->score_samples($rows);
750 6         95 my @desc = sort { $b <=> $a } @$scores;
  13458         16606  
751 6         17 my $n_pts = scalar @desc;
752 6         27 my $k = int( $self->{contamination} * $n_pts + 0.5 );
753 6 50       33 $k = 1 if $k < 1;
754 6 50       58 $k = $n_pts if $k > $n_pts;
755 6         49 $self->{threshold} = Algorithm::Classifier::IsolationForest::_threshold_from_ranked( \@desc, $k );
756 6         92 return $self;
757             } ## end sub relearn_threshold
758              
759             =head2 decision_threshold
760              
761             The score cutoff the predict methods use by default; undef unless
762             C was set and a predict-family method or
763             L has run.
764              
765             =cut
766              
767 43     43 1 3413 sub decision_threshold { return $_[0]->{threshold} }
768              
769             =head2 feature_names
770              
771             Returns the arrayref of feature name strings stored with the model, or
772             undef if none were provided.
773              
774             =cut
775              
776 2     2 1 12 sub feature_names { return $_[0]->{feature_names} }
777              
778             =head2 schema_version
779              
780             Returns the user-owned schema version string stored with the model
781             (usually via a prototype -- see PROTOTYPES in
782             L), or undef if none was
783             recorded.
784              
785             =cut
786              
787 2     2 1 1014 sub schema_version { return $_[0]->{schema_version} }
788              
789             =head2 schema_description
790              
791             Returns the free-text description of the variable schema stored with the
792             model, or undef if none was recorded.
793              
794             =cut
795              
796 0     0 1 0 sub schema_description { return $_[0]->{schema_description} }
797              
798             =head2 feature_descriptions
799              
800             Returns the hashref of per-feature description strings stored with the
801             model, or undef if none were recorded. Keys are feature names; coverage
802             may be partial.
803              
804             =cut
805              
806 3     3 1 3737 sub feature_descriptions { return $_[0]->{feature_descriptions} }
807              
808             =head2 window_count
809              
810             Returns how many points the model currently retains in its sliding
811             window (0 when C<< window_size => 0 >>).
812              
813             =cut
814              
815 22     22 1 16817 sub window_count { return scalar @{ $_[0]->{window} } }
  22         186  
816              
817             =head2 seen
818              
819             Returns the total number of points learned over the model's lifetime,
820             including points that have since been forgotten.
821              
822             =cut
823              
824 36     36 1 5026 sub seen { return $_[0]->{seen} }
825              
826             =head2 tagged_row_to_array(\%row, $caller)
827              
828             Validates a hashref of named feature values against the model's stored
829             C and returns a positional arrayref. Identical semantics
830             to the parent class's method of the same name (to which it delegates);
831             see there for the croak conditions.
832              
833             =cut
834              
835             sub tagged_row_to_array {
836 280     280 1 342 my $self = shift;
837 280         512 return Algorithm::Classifier::IsolationForest::tagged_row_to_array( $self, @_ );
838             }
839              
840             =head2 munge_rows(\@rows)
841              
842             Applies the model's scalar mungers to positional rows, exactly as the
843             parent class's method of the same name (to which it delegates); a model
844             without C returns the input unchanged.
845              
846             =cut
847              
848             sub munge_rows {
849 2     2 1 5 my $self = shift;
850 2         10 return Algorithm::Classifier::IsolationForest::munge_rows( $self, @_ );
851             }
852              
853             =head1 MODEL SAVE/LOAD METHODS
854              
855             Persistence keeps the sliding window alongside the trees, so a reloaded
856             model continues forgetting correctly as the stream resumes. This makes
857             saved online models larger than batch models by O(window_size *
858             n_features). Perl's RNG state is not persisted: a save/reload point
859             breaks bit-for-bit reproducibility of subsequent learning versus an
860             uninterrupted run, though scoring of the reloaded model is exact.
861              
862             =head2 to_json
863              
864             Returns a JSON representation of the model.
865              
866             my $json = $oif->to_json;
867              
868             =cut
869              
870             sub to_json {
871 31     31 1 342 my ($self) = @_;
872             my $payload = {
873             format => 'Algorithm::Classifier::IsolationForest::Online',
874             version => 1,
875             params => {
876             n_trees => $self->{n_trees},
877             window_size => $self->{window_size},
878             max_leaf_samples => $self->{max_leaf_samples},
879             growth => $self->{growth},
880             subsample => $self->{subsample},
881             contamination => $self->{contamination},
882             threshold => $self->{threshold},
883             n_features => $self->{n_features},
884             missing => $self->{missing},
885             feature_names => $self->{feature_names},
886             seen => $self->{seen},
887             mungers => $self->{mungers},
888             munger_module_version => $self->{munger_module_version},
889             schema_version => $self->{schema_version},
890             schema_description => $self->{schema_description},
891             feature_descriptions => $self->{feature_descriptions},
892             },
893 610         1696 trees => [ map { { count => $_->{count}, root => $_->{root} } } @{ $self->{trees} } ],
  31         200  
894             window => $self->{window},
895 31         591 };
896 31         491 return JSON::PP->new->canonical(1)->encode($payload);
897             } ## end sub to_json
898              
899             =head2 from_json($json)
900              
901             Init the object from the model in the specified JSON string.
902              
903             my $oif = Algorithm::Classifier::IsolationForest::Online->from_json($json);
904              
905             =cut
906              
907             sub from_json {
908 18     18 1 9537 my ( $class, $text ) = @_;
909 18         144 my $payload = JSON::PP->new->decode($text);
910             croak "not an online IsolationForest model"
911             unless ref $payload eq 'HASH'
912             && defined $payload->{format}
913 18 100 33     1918571 && $payload->{format} eq 'Algorithm::Classifier::IsolationForest::Online';
      66        
914              
915 16   50     132 my $p = $payload->{params} || {};
916              
917             my $self = {
918             n_trees => $p->{n_trees},
919             window_size => $p->{window_size} // 0,
920             max_leaf_samples => $p->{max_leaf_samples},
921             growth => $p->{growth} // 'adaptive',
922             subsample => $p->{subsample} // 1.0,
923             seed => undef,
924             contamination => $p->{contamination},
925             threshold => $p->{threshold},
926             n_features => $p->{n_features},
927             missing => $p->{missing} // 'die',
928             feature_names => $p->{feature_names},
929             # Recompiled lazily on first tagged use, like the parent.
930             mungers => $p->{mungers},
931             munger_module_version => $p->{munger_module_version},
932             # Opaque schema metadata; absent in models saved before prototype
933             # support, which just means "none recorded".
934             schema_version => $p->{schema_version},
935             schema_description => $p->{schema_description},
936             feature_descriptions => $p->{feature_descriptions},
937             seen => $p->{seen} // 0,
938 16   50     646 window => $payload->{window} // [],
      50        
      50        
      50        
      50        
      50        
939             trees => [],
940             _use_c => $Algorithm::Classifier::IsolationForest::HAS_C,
941             _use_openmp => $Algorithm::Classifier::IsolationForest::HAS_OPENMP,
942             };
943              
944 16         45 my $trees = $payload->{trees};
945 16 50 33     147 croak "model contains no trees" unless ref $trees eq 'ARRAY' && @$trees;
946              
947 16         79 my $model = bless $self, $class;
948              
949             # depth_limit is a pure function of the tree's count, so recompute it
950             # rather than trusting a stored float.
951             $self->{trees}
952 16         62 = [ map { { count => $_->{count}, root => $_->{root}, depth_limit => $model->_rpl( $_->{count} ) } } @$trees ];
  400         804  
953              
954 16         1669 return $model;
955             } ## end sub from_json
956              
957             =head2 save($path)
958              
959             Saves the model to the specified path.
960              
961             $oif->save($path);
962              
963             =cut
964              
965             sub save {
966 11     11 1 936 my ( $self, $path ) = @_;
967 11         74 write_file( $path, { 'atomic' => 1 }, $self->to_json );
968             }
969              
970             =head2 load($path)
971              
972             Init the object from the model in the specified file.
973              
974             my $oif = Algorithm::Classifier::IsolationForest::Online->load($path);
975              
976             =cut
977              
978             sub load {
979 2     2 1 108197 my ( $class, $path ) = @_;
980 2         11 my $raw_model = read_file($path);
981 2         253 return $class->from_json($raw_model);
982             }
983              
984             =head2 to_prototype
985              
986             Returns a prototype JSON string extracted from this model: its variable
987             schema (feature_names, feature_descriptions, mungers, missing policy)
988             plus its current tuning knobs, with C<"class": "online">. Identical
989             semantics to the parent class's method -- see PROTOTYPES in
990             L for the file format and the
991             croak/placeholder rules. C is not emitted; pass it as an override
992             when creating from the prototype.
993              
994             my $proto_json = $oif->to_prototype;
995              
996             =cut
997              
998             sub to_prototype {
999 1     1 1 9 my ($self) = @_;
1000             croak "to_prototype: this model has no feature_names; a prototype's variable " . "schema needs named features"
1001 1 50 33     7 unless ref $self->{feature_names} eq 'ARRAY' && @{ $self->{feature_names} };
  1         17  
1002              
1003             my $schema = {
1004             feature_names => $self->{feature_names},
1005             missing => $self->{missing},
1006 1         4 };
1007             $schema->{feature_descriptions} = $self->{feature_descriptions}
1008 1 50 33     4 if ref $self->{feature_descriptions} eq 'HASH' && %{ $self->{feature_descriptions} };
  1         4  
1009             $schema->{mungers} = $self->{mungers}
1010 1 50 33     4 if ref $self->{mungers} eq 'HASH' && %{ $self->{mungers} };
  0         0  
1011              
1012             my $params = {
1013             n_trees => $self->{n_trees},
1014             window_size => $self->{window_size},
1015             max_leaf_samples => $self->{max_leaf_samples},
1016             growth => $self->{growth},
1017             subsample => $self->{subsample},
1018 1         4 };
1019 1 50       3 $params->{contamination} = $self->{contamination} if defined $self->{contamination};
1020              
1021             return JSON::PP->new->canonical(1)->encode(
1022             {
1023             format => 'Algorithm::Classifier::IsolationForest::Prototype',
1024             version => 1,
1025             class => 'online',
1026             schema_version => $self->{schema_version} // '0',
1027             schema_description => $self->{schema_description}
1028 1   50     8 // '(none recorded; describe this schema and bump schema_version)',
      50        
1029             schema => $schema,
1030             params => $params,
1031             }
1032             );
1033             } ## end sub to_prototype
1034              
1035             =head1 REFERENCES
1036              
1037             Filippo Leveni, Guilherme Weigert Cassales, Bernhard Pfahringer, Albert
1038             Bifet, Giacomo Boracchi (2024). Online Isolation Forest.
1039              
1040             L
1041              
1042             L
1043              
1044             L
1045              
1046             =cut
1047              
1048             ###
1049             ###
1050             ### internal stuff below
1051             ###
1052             ###
1053              
1054             sub _check_learned {
1055 97     97   189 my ($self) = @_;
1056             croak "model has not learned any data yet; call learn() first"
1057 97 100       602 unless $self->{seen} > 0;
1058             }
1059              
1060             # Validate one incoming sample, apply the missing-value strategy, and
1061             # return a fresh dense copy (the window owns its rows; the caller may
1062             # reuse or mutate the original). Locks in n_features on first contact.
1063             sub _prep_row {
1064 16418     16418   25665 my ( $self, $row, $caller ) = @_;
1065 16418 50 33     44247 croak "$caller: each sample must be an arrayref of features"
1066             unless ref $row eq 'ARRAY' && @$row;
1067              
1068 16418 100       34800 if ( !defined $self->{n_features} ) {
    100          
1069 48         126 $self->{n_features} = scalar @$row;
1070             } elsif ( scalar @$row != $self->{n_features} ) {
1071 2         409 croak "$caller: sample has " . scalar(@$row) . " features but model expects " . $self->{n_features};
1072             }
1073              
1074 16416 100       27457 if ( $self->{missing} eq 'die' ) {
1075 15366         24247 for my $f ( 0 .. $#$row ) {
1076 31768 100       51315 next if defined $row->[$f];
1077 1         98 croak "$caller: undef feature value at column $f; "
1078             . "construct with missing => 'zero' to learn from data with missing values";
1079             }
1080 15365         31750 return [@$row];
1081             }
1082              
1083             # zero: a missing cell counts as the value 0.
1084 1050   100     2145 return [ map { $_ // 0 } @$row ];
  2100         6485  
1085             } ## end sub _prep_row
1086              
1087             # The depth budget for n points: how deep a tree fed n points is allowed
1088             # (learn) or expected (scoring normalisation, per-leaf adjustment) to
1089             # go. log base 4 = log(2 * branching_factor) with binary trees. Under
1090             # max_leaf_samples points there is nothing to isolate: 0.
1091             sub _rpl {
1092 111293     111293   143150 my ( $self, $n ) = @_;
1093 111293         137724 my $eta = $self->{max_leaf_samples};
1094 111293 100       157014 return 0 if $n < $eta;
1095 105613         175911 return log( $n / $eta ) / _LOG4;
1096             }
1097              
1098             # How many points a node at $depth needs before it may split (or below
1099             # which, on forgetting, it collapses back into a leaf).
1100             sub _split_threshold {
1101 86917     86917   114639 my ( $self, $depth ) = @_;
1102 86917 100       206928 return $self->{max_leaf_samples} * ( $self->{growth} eq 'adaptive' ? 2**$depth : 1 );
1103             }
1104              
1105             # Number of points the model currently reflects: the window fill, or the
1106             # whole stream when forgetting is disabled.
1107             sub _data_size {
1108 2006     2006   2809 my ($self) = @_;
1109 2006 100       3716 return $self->{window_size} ? scalar @{ $self->{window} } : $self->{seen};
  1904         4253  
1110             }
1111              
1112             # exp() multiplier turning a per-sample depth SUM into the normalised
1113             # anomaly score: 2**(-(sum/t)/norm) == exp(-sum * log(2)/(t*norm)).
1114             # _EPS keeps a zero normaliser (fewer than max_leaf_samples points seen)
1115             # well-defined; every depth is 0 then, so everything scores 1.0.
1116             sub _score_inv {
1117 2006     2006   2958 my ($self) = @_;
1118 2006         3139 my $norm = $self->_rpl( $self->_data_size * $self->{subsample} );
1119 2006         5411 return _LOG2 / ( $self->{n_trees} * ( $norm + _EPS ) );
1120             }
1121              
1122             #-------------------------------------------------------------------------------
1123             # Learning.
1124             #-------------------------------------------------------------------------------
1125              
1126             # Advance the stream by one (already prepped) row: every tree learns it
1127             # (subject to subsampling), it enters the window, and the oldest point
1128             # beyond the window is forgotten. This is the single choke point through
1129             # which every tree mutation flows, so it is also where the packed C
1130             # scoring snapshot gets invalidated.
1131             #
1132             # With use_c the per-tree learn and eviction loops run inside the
1133             # parent's C backend (online_learn_row_xs / online_unlearn_row_xs),
1134             # mutating the same live trees this file's Perl recursion would. Random
1135             # draws go through the same generator in the same order, so the trees
1136             # built are bit-identical either way (on nvsize == 8 perls) -- use_c
1137             # only changes speed, matching fit()'s guarantee.
1138             sub _learn_row {
1139 16415     16415   23627 my ( $self, $r ) = @_;
1140 16415         21752 my $sub = $self->{subsample};
1141              
1142 16415         31262 $self->_invalidate_c_trees;
1143              
1144 16415 100       24050 if ( _HAS_ONLINE_XS && $self->{_use_c} ) {
1145             Algorithm::Classifier::IsolationForest::online_learn_row_xs(
1146             $self->{trees}, $r, $self->{n_features},
1147             $self->{max_leaf_samples},
1148 13415 100       337326 ( $self->{growth} eq 'adaptive' ? 1 : 0 ), $sub
1149             );
1150             } else {
1151 3000         3958 for my $tree ( @{ $self->{trees} } ) {
  3000         4980  
1152 46500 100 100     79023 next if $sub < 1 && rand() >= $sub;
1153 44471         62529 $self->_tree_learn( $tree, $r );
1154             }
1155             }
1156 16415         24387 $self->{seen}++;
1157              
1158 16415 100       27513 if ( $self->{window_size} ) {
1159 15115         17682 push @{ $self->{window} }, $r;
  15115         23659  
1160 15115 100       18812 if ( @{ $self->{window} } > $self->{window_size} ) {
  15115         26372  
1161 8463         10213 my $old = shift @{ $self->{window} };
  8463         12103  
1162 8463 100       13858 if ( _HAS_ONLINE_XS && $self->{_use_c} ) {
1163             Algorithm::Classifier::IsolationForest::online_unlearn_row_xs(
1164             $self->{trees}, $old, $self->{n_features},
1165             $self->{max_leaf_samples},
1166 7003 100       407887 ( $self->{growth} eq 'adaptive' ? 1 : 0 ), $sub
1167             );
1168             } else {
1169 1460         1957 for my $tree ( @{ $self->{trees} } ) {
  1460         2386  
1170 22760 100 100     40718 next if $sub < 1 && rand() >= $sub;
1171 21270         42846 $self->_tree_unlearn( $tree, $old );
1172             }
1173             }
1174             } ## end if ( @{ $self->{window} } > $self->{window_size...})
1175             } ## end if ( $self->{window_size} )
1176 16415         29342 return;
1177             } ## end sub _learn_row
1178              
1179             sub _tree_learn {
1180 44471     44471   60105 my ( $self, $tree, $x ) = @_;
1181 44471         55413 $tree->{count}++;
1182 44471         66600 $tree->{depth_limit} = $self->_rpl( $tree->{count} );
1183 44471 100       65214 if ( !defined $tree->{root} ) {
1184 110         223 $tree->{root} = [ _NT_LEAF, 1, [@$x], [@$x] ];
1185             } else {
1186 44361         65636 $tree->{root} = $self->_node_learn( $tree->{root}, $x, 0, $tree->{depth_limit} );
1187             }
1188 44471         61660 return;
1189             } ## end sub _tree_learn
1190              
1191             # Route $x down to its leaf, growing counts and bounding boxes along the
1192             # path. A leaf that has accumulated its split requirement (and still has
1193             # depth budget) is replaced by a subtree built from synthetic points
1194             # sampled inside its box -- the return value replaces the node in the
1195             # parent, which is how leaves turn into subtrees in place.
1196             sub _node_learn {
1197 125393     125393   174724 my ( $self, $node, $x, $depth, $limit ) = @_;
1198              
1199 125393         144955 $node->[_N_COUNT]++;
1200 125393 50       163888 if ( !defined $node->[_N_LO] ) {
1201              
1202             # Leaf born from an empty synthetic partition: first real point
1203             # initialises the box.
1204 0         0 $node->[_N_LO] = [@$x];
1205 0         0 $node->[_N_HI] = [@$x];
1206             } else {
1207 125393         164541 my ( $lo, $hi ) = ( $node->[_N_LO], $node->[_N_HI] );
1208 125393         176330 for my $f ( 0 .. $#$x ) {
1209 250786         287220 my $v = $x->[$f];
1210 250786 100       353755 $lo->[$f] = $v if $v < $lo->[$f];
1211 250786 100       393731 $hi->[$f] = $v if $v > $hi->[$f];
1212             }
1213             }
1214              
1215 125393 100       185054 if ( $node->[_N_TYPE] == _NT_LEAF ) {
1216 44361 100 100     65667 if ( $node->[_N_COUNT] >= $self->_split_threshold($depth) && $depth < $limit ) {
1217 449         987 my $pts = $self->_sample_box( $node->[_N_LO], $node->[_N_HI], $node->[_N_COUNT] );
1218 449         964 return $self->_build_from_points( $pts, $depth, $limit );
1219             }
1220 43912         65179 return $node;
1221             }
1222              
1223 81032 100       121162 my $ci = $x->[ $node->[_N_ATTR] ] < $node->[_N_SPLIT] ? _N_LEFT : _N_RIGHT;
1224 81032         121198 $node->[$ci] = $self->_node_learn( $node->[$ci], $x, $depth + 1, $limit );
1225 81032         114056 return $node;
1226             } ## end sub _node_learn
1227              
1228             # $n synthetic points drawn uniformly inside the box -- the stand-in for
1229             # the real points the tree never stored.
1230             sub _sample_box {
1231 449     449   782 my ( $self, $lo, $hi, $n ) = @_;
1232 449         652 my @pts;
1233 449         798 for ( 1 .. $n ) {
1234 15686 50       20873 push @pts, [ map { my $w = $hi->[$_] - $lo->[$_]; $w > 0 ? $lo->[$_] + rand() * $w : $lo->[$_] } 0 .. $#$lo ];
  31372         37620  
  31372         56529  
1235             }
1236 449         809 return \@pts;
1237             }
1238              
1239             # Recursively build a subtree over (synthetic) points: random feature,
1240             # uniform split value within the points' range on it, recurse on the
1241             # partitions. Leaves keep the partition's count and box.
1242             sub _build_from_points {
1243 1347     1347   2304 my ( $self, $pts, $depth, $limit ) = @_;
1244 1347         1726 my $n = scalar @$pts;
1245 1347         2064 my ( $lo, $hi ) = _box_of($pts);
1246              
1247 1347 100 100     2465 if ( $n < $self->_split_threshold($depth) || $depth >= $limit ) {
1248 898         2097 return [ _NT_LEAF, $n, $lo, $hi ];
1249             }
1250              
1251 449         980 my $attr = int( rand( $self->{n_features} ) );
1252 449         842 my ( $pmin, $pmax ) = ( $pts->[0][$attr], $pts->[0][$attr] );
1253 449         720 for my $p (@$pts) {
1254 15686 100       21469 $pmin = $p->[$attr] if $p->[$attr] < $pmin;
1255 15686 100       22693 $pmax = $p->[$attr] if $p->[$attr] > $pmax;
1256             }
1257 449         757 my $split = $pmin + rand() * ( $pmax - $pmin );
1258              
1259 449         652 my ( @l, @r );
1260 449         642 for my $p (@$pts) {
1261 15686 100       19891 if ( $p->[$attr] < $split ) { push @l, $p }
  7769         9579  
1262 7917         10484 else { push @r, $p }
1263             }
1264              
1265 449         1001 my $left = $self->_build_from_points( \@l, $depth + 1, $limit );
1266 449         938 my $right = $self->_build_from_points( \@r, $depth + 1, $limit );
1267 449         3712 return [ _NT_AXIS, $n, $lo, $hi, $attr, $split, $left, $right ];
1268             } ## end sub _build_from_points
1269              
1270             #-------------------------------------------------------------------------------
1271             # Forgetting.
1272             #-------------------------------------------------------------------------------
1273              
1274             sub _tree_unlearn {
1275 21270     21270   30699 my ( $self, $tree, $x ) = @_;
1276 21270         27842 $tree->{count}--;
1277 21270         32638 $tree->{depth_limit} = $self->_rpl( $tree->{count} );
1278 21270 50       34824 return unless defined $tree->{root};
1279 21270         32670 $tree->{root} = $self->_node_unlearn( $tree->{root}, $x, 0 );
1280 21270         31705 return;
1281             }
1282              
1283             # Route the forgotten point down its (current) path, decrementing counts.
1284             # An internal node whose count no longer justifies its split collapses
1285             # back into a leaf; otherwise its box is refreshed to the union of its
1286             # children's, which is how boxes shrink as old extremes age out.
1287             sub _node_unlearn {
1288 62358     62358   88615 my ( $self, $node, $x, $depth ) = @_;
1289              
1290 62358         75652 $node->[_N_COUNT]--;
1291 62358 100       100181 return $node if $node->[_N_TYPE] == _NT_LEAF;
1292 41209 100       60135 return _collapse($node) if $node->[_N_COUNT] < $self->_split_threshold($depth);
1293              
1294 41088 100       66558 my $ci = $x->[ $node->[_N_ATTR] ] < $node->[_N_SPLIT] ? _N_LEFT : _N_RIGHT;
1295 41088         62139 $node->[$ci] = $self->_node_unlearn( $node->[$ci], $x, $depth + 1 );
1296              
1297 41088         61348 my ( $lo, $hi ) = _box_union( $node->[_N_LEFT], $node->[_N_RIGHT] );
1298 41088 50       62377 if ( defined $lo ) {
1299 41088         57411 $node->[_N_LO] = $lo;
1300 41088         54564 $node->[_N_HI] = $hi;
1301             }
1302 41088         60272 return $node;
1303             } ## end sub _node_unlearn
1304              
1305             # Aggregate a subtree back into a single leaf holding the subtree's
1306             # (already decremented) count and the union of its descendants' boxes.
1307             sub _collapse {
1308 363     363   484 my ($node) = @_;
1309 363 100       727 return $node if $node->[_N_TYPE] == _NT_LEAF;
1310 121         226 my $l = _collapse( $node->[_N_LEFT] );
1311 121         248 my $r = _collapse( $node->[_N_RIGHT] );
1312 121         245 my ( $lo, $hi ) = _box_union( $l, $r );
1313 121 50       235 if ( !defined $lo ) {
1314              
1315             # Both children empty: keep the node's own box.
1316 0         0 ( $lo, $hi ) = ( $node->[_N_LO], $node->[_N_HI] );
1317             }
1318 121         532 return [ _NT_LEAF, $node->[_N_COUNT], $lo, $hi ];
1319             } ## end sub _collapse
1320              
1321             # (lo, hi) of the union of two nodes' boxes, as fresh arrays (parent
1322             # boxes grow in place, so they must never alias a child's). Nodes with
1323             # no box yet (empty leaves) are skipped; (undef, undef) if neither has
1324             # one.
1325             sub _box_union {
1326 41209     41209   54079 my ( $a, $b ) = @_;
1327 41209         55304 my @boxed = grep { defined $_->[_N_LO] } ( $a, $b );
  82418         124089  
1328 41209 50       58568 return ( undef, undef ) unless @boxed;
1329 41209         46233 my $lo = [ @{ $boxed[0][_N_LO] } ];
  41209         61462  
1330 41209         48849 my $hi = [ @{ $boxed[0][_N_HI] } ];
  41209         60213  
1331 41209 50       66043 if ( @boxed == 2 ) {
1332 41209         57574 my ( $blo, $bhi ) = ( $boxed[1][_N_LO], $boxed[1][_N_HI] );
1333 41209         61417 for my $f ( 0 .. $#$lo ) {
1334 82418 100       126956 $lo->[$f] = $blo->[$f] if $blo->[$f] < $lo->[$f];
1335 82418 100       140833 $hi->[$f] = $bhi->[$f] if $bhi->[$f] > $hi->[$f];
1336             }
1337             }
1338 41209         69647 return ( $lo, $hi );
1339             } ## end sub _box_union
1340              
1341             # (lo, hi) bounding box of a point set; (undef, undef) when empty.
1342             sub _box_of {
1343 1347     1347   1858 my ($pts) = @_;
1344 1347 50       2327 return ( undef, undef ) unless @$pts;
1345 1347         1589 my $lo = [ @{ $pts->[0] } ];
  1347         2395  
1346 1347         1715 my $hi = [ @{ $pts->[0] } ];
  1347         2195  
1347 1347         1948 for my $p (@$pts) {
1348 31372         42056 for my $f ( 0 .. $#$p ) {
1349 62744 100       89080 $lo->[$f] = $p->[$f] if $p->[$f] < $lo->[$f];
1350 62744 100       101196 $hi->[$f] = $p->[$f] if $p->[$f] > $hi->[$f];
1351             }
1352             }
1353 1347         2620 return ( $lo, $hi );
1354             } ## end sub _box_of
1355              
1356             #-------------------------------------------------------------------------------
1357             # Scoring.
1358             #-------------------------------------------------------------------------------
1359              
1360             # Depth of the leaf $x lands in, plus the leaf's own depth budget -- the
1361             # streaming analogue of the batch scorer's c(leaf size) adjustment.
1362             # Scoring tolerates undef cells (mapped to 0), matching the parent class.
1363             sub _depth_of {
1364 39080     39080   51022 my ( $self, $x, $node ) = @_;
1365 39080         42268 my $depth = 0;
1366 39080         52414 while ( $node->[_N_TYPE] ) {
1367 98226 100 100     154877 $node = ( $x->[ $node->[_N_ATTR] ] // 0 ) < $node->[_N_SPLIT] ? $node->[_N_LEFT] : $node->[_N_RIGHT];
1368 98226         131040 $depth++;
1369             }
1370 39080         51329 return $depth + $self->_rpl( $node->[_N_COUNT] );
1371             }
1372              
1373             # Per-sample depth sums across all trees (tree-outer, sample-inner for
1374             # cache locality, mirroring the parent's pure-Perl loops).
1375             sub _depth_sums {
1376 14     14   40 my ( $self, $data ) = @_;
1377 14         78 my @sums = (0) x @$data;
1378 14         42 for my $tree ( @{ $self->{trees} } ) {
  14         32  
1379 700         1047 my $root = $tree->{root};
1380 700 50       965 next unless defined $root;
1381 700         955 for my $i ( 0 .. $#$data ) {
1382 28600         38660 $sums[$i] += $self->_depth_of( $data->[$i], $root );
1383             }
1384             }
1385 14         34 return \@sums;
1386             } ## end sub _depth_sums
1387              
1388             # Single-row score against the current model state; used by the
1389             # prequential score_learn loop, where the normaliser moves as points are
1390             # learned and so must be recomputed per row.
1391             sub _score_row {
1392 1924     1924   2794 my ( $self, $r ) = @_;
1393 1924 100       3261 if ( _HAS_ONLINE_XS && $self->{_use_c} ) {
1394              
1395             # Walks the live trees in C -- no packed snapshot involved, so
1396             # this stays fast even though score_learn mutates the trees
1397             # between rows.
1398             my $sum = Algorithm::Classifier::IsolationForest::online_score_row_xs( $self->{trees}, $r,
1399 1324         8823 $self->{n_features}, $self->{max_leaf_samples} );
1400 1324         2327 return exp( -$sum * $self->_score_inv );
1401             }
1402 600         823 my $sum = 0;
1403 600         704 for my $tree ( @{ $self->{trees} } ) {
  600         922  
1404 10500 100       20627 $sum += $self->_depth_of( $r, $tree->{root} ) if defined $tree->{root};
1405             }
1406 600         1235 return exp( -$sum * $self->_score_inv );
1407             } ## end sub _score_row
1408              
1409             #-------------------------------------------------------------------------------
1410             # C-accelerated scoring.
1411             #
1412             # The parent class's Inline::C scorer walks immutable packed node buffers;
1413             # online trees mutate on every learned point. The bridge is a lazily
1414             # built snapshot: the first scoring call after any mutation flattens the
1415             # live trees into the parent's packed node layout (below) and every
1416             # scoring call until the next mutation reuses it. _learn_row -- the one
1417             # choke point all mutations flow through -- drops the snapshot.
1418             #
1419             # Online trees are axis-only, so they map onto the parent's 6-double node
1420             # records directly:
1421             #
1422             # leaf: [0, count, _rpl(count), 0, 0, 0]
1423             # axis: [1, attr, split, li, ri, 0]
1424             #
1425             # The parent packs c(leaf size) into slot 2 and its C walker returns
1426             # depth + slot2 at a leaf; packing the online depth-budget adjustment
1427             # _rpl(count) there instead makes score_all_xs compute exactly the
1428             # pure-Perl _depth_of value, so every downstream C helper (finalize_*,
1429             # predict_sums_xs, score_predict_*) applies unchanged. The per-tree
1430             # coefficient buffers are empty -- there are no oblique nodes -- and only
1431             # exist because score_all_xs expects them.
1432             #
1433             # score_learn deliberately never uses this path: it mutates the trees
1434             # after every single point, so the snapshot could never be reused and
1435             # repacking per point would cost more than the walks it replaces.
1436             #-------------------------------------------------------------------------------
1437              
1438             # Drop the packed snapshot; called on every mutation.
1439             sub _invalidate_c_trees {
1440 16415     16415   19420 delete @{ $_[0] }{qw(_c_nodes _c_coef_idx _c_coef_val)};
  16415         32630  
1441 16415         19814 return;
1442             }
1443              
1444             # Build (or reuse) the packed snapshot. Returns true when the C scoring
1445             # path may be taken, false when the caller must use the pure-Perl walk.
1446             sub _ensure_c_trees {
1447 92     92   202 my ($self) = @_;
1448 92 100       523 return 0 unless $self->{_use_c};
1449 71 100       353 return 1 if $self->{_c_nodes};
1450              
1451 27         53 my ( @c_nodes, @c_coef_idx, @c_coef_val );
1452 27         51 my $empty_idx = pack('l*');
1453 27         52 my $empty_val = pack('d*');
1454 27         40 for my $tree ( @{ $self->{trees} } ) {
  27         74  
1455 905         1551 push @c_nodes, $self->_pack_online_tree( $tree->{root} );
1456 905         1371 push @c_coef_idx, $empty_idx;
1457 905         1472 push @c_coef_val, $empty_val;
1458             }
1459 27         101 $self->{_c_nodes} = \@c_nodes;
1460 27         149 $self->{_c_coef_idx} = \@c_coef_idx;
1461 27         85 $self->{_c_coef_val} = \@c_coef_val;
1462 27         104 return 1;
1463             } ## end sub _ensure_c_trees
1464              
1465             # Flatten one live tree into the parent's packed node buffer (DFS
1466             # pre-order, root at index 0 -- the origin score_all_xs walks from).
1467             sub _pack_online_tree {
1468 905     905   1278 my ( $self, $root ) = @_;
1469              
1470             # A tree that has not learned anything walks as depth 0 with a zero
1471             # adjustment: one empty leaf record.
1472 905 50       1430 return pack( 'd*', 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ) unless defined $root;
1473              
1474 905         1143 my @node_data;
1475             my $assign;
1476             $assign = sub {
1477 7227     7227   8987 my ($node) = @_;
1478 7227         8275 my $my_idx = scalar @node_data;
1479 7227         9088 push @node_data, undef; # reserve slot; filled in after children
1480 7227 100       10031 if ( $node->[_N_TYPE] == _NT_LEAF ) {
1481 4066         5990 $node_data[$my_idx]
1482             = [ 0.0, $node->[_N_COUNT] + 0.0, $self->_rpl( $node->[_N_COUNT] ) + 0.0, 0.0, 0.0, 0.0 ];
1483             } else {
1484 3161         7428 my $li = $assign->( $node->[_N_LEFT] );
1485 3161         4168 my $ri = $assign->( $node->[_N_RIGHT] );
1486 3161         5949 $node_data[$my_idx]
1487             = [ 1.0, $node->[_N_ATTR] + 0.0, $node->[_N_SPLIT] + 0.0, $li + 0.0, $ri + 0.0, 0.0 ];
1488             }
1489 7227         9184 return $my_idx;
1490 905         2951 }; ## end $assign = sub
1491 905         1779 $assign->($root);
1492 905         1284 return pack( 'd*', map { @$_ } @node_data );
  7227         15242  
1493             } ## end sub _pack_online_tree
1494              
1495             # Pack the query rows into the row-major double buffer score_all_xs
1496             # reads, via the parent's C row walker. miss_mode 0 maps an undef cell
1497             # to 0.0, matching the pure-Perl walk's "// 0".
1498             sub _pack_input {
1499 71     71   134 my ( $self, $data ) = @_;
1500 71         142 my $n_pts = scalar @$data;
1501 71         157 my $nf = $self->{n_features};
1502 71         292 my $x_packed = "\0" x ( $n_pts * $nf * 8 );
1503 71         744 Algorithm::Classifier::IsolationForest::pack_input_xs( $data, $x_packed, $n_pts, $nf, 0, '' );
1504 71         195 return ( $n_pts, $x_packed );
1505             }
1506              
1507             # Lazily learn the contamination threshold from the current window the
1508             # first time a predict-family method needs it. A model with no retained
1509             # window (window_size 0) stays on the 0.5 fallback until the caller runs
1510             # relearn_threshold with data.
1511             sub _ensure_threshold {
1512 23     23   84 my ($self) = @_;
1513             return
1514             if !defined $self->{contamination}
1515             || defined $self->{threshold}
1516 23 100 100     117 || !@{ $self->{window} };
  4   66     16  
1517 4         22 $self->relearn_threshold;
1518 4         11 return;
1519             }
1520              
1521             1;