| line |
stmt |
bran |
cond |
sub |
pod |
time |
code |
|
1
|
|
|
|
|
|
|
package Algorithm::Classifier::IsolationForest::Online; |
|
2
|
|
|
|
|
|
|
|
|
3
|
90
|
|
|
90
|
|
200049
|
use strict; |
|
|
90
|
|
|
|
|
152
|
|
|
|
90
|
|
|
|
|
3668
|
|
|
4
|
90
|
|
|
90
|
|
1125
|
use warnings; |
|
|
90
|
|
|
|
|
355
|
|
|
|
90
|
|
|
|
|
4213
|
|
|
5
|
90
|
|
|
90
|
|
486
|
use Carp qw(croak); |
|
|
90
|
|
|
|
|
238
|
|
|
|
90
|
|
|
|
|
4213
|
|
|
6
|
90
|
|
|
90
|
|
1019
|
use JSON::PP (); |
|
|
90
|
|
|
|
|
15225
|
|
|
|
90
|
|
|
|
|
2016
|
|
|
7
|
90
|
|
|
90
|
|
1188
|
use File::Slurp qw(read_file write_file); |
|
|
90
|
|
|
|
|
38203
|
|
|
|
90
|
|
|
|
|
4079
|
|
|
8
|
|
|
|
|
|
|
|
|
9
|
|
|
|
|
|
|
# Runtime-only dependency: tagged_row_to_array is delegated to the parent |
|
10
|
|
|
|
|
|
|
# class (identical semantics, no point duplicating it) and the |
|
11
|
|
|
|
|
|
|
# contamination threshold selection reuses _threshold_from_ranked. The |
|
12
|
|
|
|
|
|
|
# parent never loads this module at compile time (its from_json requires |
|
13
|
|
|
|
|
|
|
# it on demand), so there is no cycle. |
|
14
|
90
|
|
|
90
|
|
2571
|
use Algorithm::Classifier::IsolationForest (); |
|
|
90
|
|
|
|
|
124
|
|
|
|
90
|
|
|
|
|
3352
|
|
|
15
|
|
|
|
|
|
|
|
|
16
|
|
|
|
|
|
|
our $VERSION = '0.6.0'; |
|
17
|
|
|
|
|
|
|
|
|
18
|
|
|
|
|
|
|
# Node layout. Unlike the batch forest's nodes, online nodes are mutable |
|
19
|
|
|
|
|
|
|
# and carry a running point count plus the bounding box (per-feature |
|
20
|
|
|
|
|
|
|
# lo/hi) of every point that has passed through them -- that box is what |
|
21
|
|
|
|
|
|
|
# split simulation samples from, since points themselves are never stored |
|
22
|
|
|
|
|
|
|
# in the tree. Both node types share the first four slots so the |
|
23
|
|
|
|
|
|
|
# learn/unlearn bookkeeping never has to branch on type: |
|
24
|
|
|
|
|
|
|
# |
|
25
|
|
|
|
|
|
|
# leaf: [0, count, \@lo, \@hi] |
|
26
|
|
|
|
|
|
|
# internal: [1, count, \@lo, \@hi, attr, split, left, right] |
|
27
|
|
|
|
|
|
|
# |
|
28
|
|
|
|
|
|
|
# The type tag mirrors the parent's convention (0 is falsy, so |
|
29
|
|
|
|
|
|
|
# while ($node->[0]) walks to a leaf). A leaf built from an empty |
|
30
|
|
|
|
|
|
|
# synthetic partition has count 0 and an undef box (slots 2/3); the box |
|
31
|
|
|
|
|
|
|
# is initialised from the first real point that reaches it. |
|
32
|
90
|
|
|
90
|
|
347
|
use constant _N_TYPE => 0; |
|
|
90
|
|
|
|
|
130
|
|
|
|
90
|
|
|
|
|
6173
|
|
|
33
|
90
|
|
|
90
|
|
404
|
use constant _N_COUNT => 1; |
|
|
90
|
|
|
|
|
198
|
|
|
|
90
|
|
|
|
|
4249
|
|
|
34
|
90
|
|
|
90
|
|
368
|
use constant _N_LO => 2; |
|
|
90
|
|
|
|
|
164
|
|
|
|
90
|
|
|
|
|
3239
|
|
|
35
|
90
|
|
|
90
|
|
367
|
use constant _N_HI => 3; |
|
|
90
|
|
|
|
|
220
|
|
|
|
90
|
|
|
|
|
3029
|
|
|
36
|
90
|
|
|
90
|
|
350
|
use constant _N_ATTR => 4; |
|
|
90
|
|
|
|
|
147
|
|
|
|
90
|
|
|
|
|
3250
|
|
|
37
|
90
|
|
|
90
|
|
391
|
use constant _N_SPLIT => 5; |
|
|
90
|
|
|
|
|
164
|
|
|
|
90
|
|
|
|
|
2912
|
|
|
38
|
90
|
|
|
90
|
|
372
|
use constant _N_LEFT => 6; |
|
|
90
|
|
|
|
|
132
|
|
|
|
90
|
|
|
|
|
2759
|
|
|
39
|
90
|
|
|
90
|
|
328
|
use constant _N_RIGHT => 7; |
|
|
90
|
|
|
|
|
150
|
|
|
|
90
|
|
|
|
|
2775
|
|
|
40
|
|
|
|
|
|
|
|
|
41
|
90
|
|
|
90
|
|
340
|
use constant _NT_LEAF => 0; |
|
|
90
|
|
|
|
|
129
|
|
|
|
90
|
|
|
|
|
2587
|
|
|
42
|
90
|
|
|
90
|
|
370
|
use constant _NT_AXIS => 1; |
|
|
90
|
|
|
|
|
165
|
|
|
|
90
|
|
|
|
|
4759
|
|
|
43
|
|
|
|
|
|
|
|
|
44
|
|
|
|
|
|
|
# Trees are binary (the reference implementation's branching_factor == 2), |
|
45
|
|
|
|
|
|
|
# which fixes the depth-budget log base at log(2 * 2). Spelled as the |
|
46
|
|
|
|
|
|
|
# exact-double literal rather than log(4) so it is bit-identical to the |
|
47
|
|
|
|
|
|
|
# OL_LOG4 literal the C learn path uses regardless of the platform's |
|
48
|
|
|
|
|
|
|
# libm rounding -- a one-ulp disagreement would flip `depth < limit` |
|
49
|
|
|
|
|
|
|
# split decisions exactly when a tree's count is eta * 4**k (the same |
|
50
|
|
|
|
|
|
|
# TWO_PI trick the parent uses for _randn parity). |
|
51
|
90
|
|
|
90
|
|
453
|
use constant _LOG4 => unpack( 'd', pack 'd', 1.3862943611198906 ); |
|
|
90
|
|
|
|
|
346
|
|
|
|
90
|
|
|
|
|
4064
|
|
|
52
|
90
|
|
|
90
|
|
351
|
use constant _LOG2 => log(2); |
|
|
90
|
|
|
|
|
145
|
|
|
|
90
|
|
|
|
|
4040
|
|
|
53
|
|
|
|
|
|
|
|
|
54
|
|
|
|
|
|
|
# DBL_EPSILON, added to the normalisation factor before dividing so a |
|
55
|
|
|
|
|
|
|
# just-started model (normaliser 0) yields well-defined scores instead of |
|
56
|
|
|
|
|
|
|
# a division by zero -- the same guard the reference implementation uses. |
|
57
|
90
|
|
|
90
|
|
368
|
use constant _EPS => 2.220446049250313e-16; |
|
|
90
|
|
|
|
|
129
|
|
|
|
90
|
|
|
|
|
4203
|
|
|
58
|
|
|
|
|
|
|
|
|
59
|
|
|
|
|
|
|
# The online learn/unlearn/score-row XS functions were added to the C |
|
60
|
|
|
|
|
|
|
# backend after the batch-scoring ones, so a prebuilt object installed |
|
61
|
|
|
|
|
|
|
# from an older release can back $HAS_C while lacking them (the parent |
|
62
|
|
|
|
|
|
|
# trusts a flag-matched prebuilt object without inspecting its symbol |
|
63
|
|
|
|
|
|
|
# set). Probe once at load: without them, use_c still accelerates the |
|
64
|
|
|
|
|
|
|
# packed-snapshot batch scoring -- those functions have been in the |
|
65
|
|
|
|
|
|
|
# object all along -- and learning quietly stays pure Perl instead of |
|
66
|
|
|
|
|
|
|
# crashing on an undefined XS sub. Rebuilding/reinstalling (or |
|
67
|
|
|
|
|
|
|
# IF_RUNTIME_BUILD=1) restores the full set. |
|
68
|
90
|
50
|
|
90
|
|
376
|
use constant _HAS_ONLINE_XS => defined &Algorithm::Classifier::IsolationForest::online_learn_row_xs ? 1 : 0; |
|
|
90
|
|
|
|
|
142
|
|
|
|
90
|
|
|
|
|
494472
|
|
|
69
|
|
|
|
|
|
|
|
|
70
|
|
|
|
|
|
|
=head1 NAME |
|
71
|
|
|
|
|
|
|
|
|
72
|
|
|
|
|
|
|
Algorithm::Classifier::IsolationForest::Online - Online (streaming) Isolation Forest anomaly detection |
|
73
|
|
|
|
|
|
|
|
|
74
|
|
|
|
|
|
|
=head1 SYNOPSIS |
|
75
|
|
|
|
|
|
|
|
|
76
|
|
|
|
|
|
|
use Algorithm::Classifier::IsolationForest::Online; |
|
77
|
|
|
|
|
|
|
|
|
78
|
|
|
|
|
|
|
my $oif = Algorithm::Classifier::IsolationForest::Online->new( |
|
79
|
|
|
|
|
|
|
n_trees => 100, |
|
80
|
|
|
|
|
|
|
window_size => 2048, |
|
81
|
|
|
|
|
|
|
max_leaf_samples => 32, |
|
82
|
|
|
|
|
|
|
seed => 42, |
|
83
|
|
|
|
|
|
|
); |
|
84
|
|
|
|
|
|
|
|
|
85
|
|
|
|
|
|
|
# stream data through the model; each point is learned and old |
|
86
|
|
|
|
|
|
|
# points beyond the window are forgotten automatically |
|
87
|
|
|
|
|
|
|
$oif->learn(\@warmup_rows); |
|
88
|
|
|
|
|
|
|
|
|
89
|
|
|
|
|
|
|
# prequential operation: score each point against the model as it |
|
90
|
|
|
|
|
|
|
# stood BEFORE that point was learned, then learn it |
|
91
|
|
|
|
|
|
|
my $scores = $oif->score_learn(\@new_rows); |
|
92
|
|
|
|
|
|
|
|
|
93
|
|
|
|
|
|
|
# or score without learning |
|
94
|
|
|
|
|
|
|
my $scores2 = $oif->score_samples(\@query_rows); |
|
95
|
|
|
|
|
|
|
my $labels = $oif->predict(\@query_rows); |
|
96
|
|
|
|
|
|
|
|
|
97
|
|
|
|
|
|
|
# persistence keeps the window, so a reloaded model keeps forgetting |
|
98
|
|
|
|
|
|
|
# correctly as the stream continues |
|
99
|
|
|
|
|
|
|
$oif->save('oiforest_model.json'); |
|
100
|
|
|
|
|
|
|
my $resumed = Algorithm::Classifier::IsolationForest::Online->load('oiforest_model.json'); |
|
101
|
|
|
|
|
|
|
|
|
102
|
|
|
|
|
|
|
=head1 DESCRIPTION |
|
103
|
|
|
|
|
|
|
|
|
104
|
|
|
|
|
|
|
Implements Online Isolation Forest (Online-iForest; Leveni, Weigert |
|
105
|
|
|
|
|
|
|
Cassales, Pfahringer, Bifet & Boracchi 2024 -- see REFERENCES), a |
|
106
|
|
|
|
|
|
|
streaming variant of Isolation Forest for data that arrives continuously |
|
107
|
|
|
|
|
|
|
and whose distribution may drift. There is no C: the model |
|
108
|
|
|
|
|
|
|
Cs points as they arrive and, once more than C points |
|
109
|
|
|
|
|
|
|
have been seen, forgets the oldest point for every new one so the model |
|
110
|
|
|
|
|
|
|
always reflects the most recent C points of the stream. |
|
111
|
|
|
|
|
|
|
|
|
112
|
|
|
|
|
|
|
Trees never store data points. Each node keeps only a running count of |
|
113
|
|
|
|
|
|
|
the points that passed through it and the bounding box of their feature |
|
114
|
|
|
|
|
|
|
values. A leaf splits once enough points have accumulated (see |
|
115
|
|
|
|
|
|
|
C and C); because the actual points are gone, |
|
116
|
|
|
|
|
|
|
the split simulates them by sampling uniformly inside the leaf's bounding |
|
117
|
|
|
|
|
|
|
box. Forgetting reverses the process: counts are decremented along the |
|
118
|
|
|
|
|
|
|
forgotten point's path and a subtree whose count falls below its split |
|
119
|
|
|
|
|
|
|
requirement is collapsed back into a leaf. |
|
120
|
|
|
|
|
|
|
|
|
121
|
|
|
|
|
|
|
Scoring follows the classic Isolation Forest intuition -- anomalies |
|
122
|
|
|
|
|
|
|
isolate at shallow depth -- but normalises by the depth budget |
|
123
|
|
|
|
|
|
|
C of the current window rather than the |
|
124
|
|
|
|
|
|
|
batch model's C. Scores are in (0, 1] with high values |
|
125
|
|
|
|
|
|
|
anomalous, directly comparable in spirit (though not numerically) to the |
|
126
|
|
|
|
|
|
|
parent class's scores. |
|
127
|
|
|
|
|
|
|
|
|
128
|
|
|
|
|
|
|
Both learning and scoring are accelerated through the parent class's |
|
129
|
|
|
|
|
|
|
Inline::C backend when it is available; C covers them together. |
|
130
|
|
|
|
|
|
|
|
|
131
|
|
|
|
|
|
|
Learning (and the per-row walks inside C) runs in C |
|
132
|
|
|
|
|
|
|
directly against the live trees, drawing randomness through the same |
|
133
|
|
|
|
|
|
|
generator in the same order as the pure-Perl path -- so, like the |
|
134
|
|
|
|
|
|
|
parent's C, a C with a given seed produces bit-identical |
|
135
|
|
|
|
|
|
|
trees whether C is on or off (on C perls; wide-NV |
|
136
|
|
|
|
|
|
|
perls keep extra low bits in the pure-Perl path). The knob changes |
|
137
|
|
|
|
|
|
|
speed, never results. |
|
138
|
|
|
|
|
|
|
|
|
139
|
|
|
|
|
|
|
Batch scoring lazily flattens the mutable trees into the same packed |
|
140
|
|
|
|
|
|
|
node layout the batch scorer walks -- online trees are axis-only, and |
|
141
|
|
|
|
|
|
|
the online per-leaf depth adjustment rides in the slot the batch packer |
|
142
|
|
|
|
|
|
|
uses for its own leaf adjustment -- so C, C, |
|
143
|
|
|
|
|
|
|
C, C, and C |
|
144
|
|
|
|
|
|
|
all run through the same C (and OpenMP, when linked) tree walk the |
|
145
|
|
|
|
|
|
|
parent uses, with identical results to the pure-Perl fallback. Any |
|
146
|
|
|
|
|
|
|
C invalidates the packed snapshot; the next batch-scoring call |
|
147
|
|
|
|
|
|
|
repacks once. C never touches the snapshot: it mutates |
|
148
|
|
|
|
|
|
|
the trees after every single point, so its rows are scored by walking |
|
149
|
|
|
|
|
|
|
the live trees in C instead. |
|
150
|
|
|
|
|
|
|
|
|
151
|
|
|
|
|
|
|
A model needs to have seen at least C points before |
|
152
|
|
|
|
|
|
|
tree structure exists at all; until then every point scores 1.0. Give |
|
153
|
|
|
|
|
|
|
the model a warm-up C pass before trusting scores or labels. |
|
154
|
|
|
|
|
|
|
|
|
155
|
|
|
|
|
|
|
Models saved by this class carry their own C tag. |
|
156
|
|
|
|
|
|
|
C<< Algorithm::Classifier::IsolationForest->load >> recognises it and |
|
157
|
|
|
|
|
|
|
dispatches here, so callers can load either model type through the |
|
158
|
|
|
|
|
|
|
parent class. |
|
159
|
|
|
|
|
|
|
|
|
160
|
|
|
|
|
|
|
=head1 GENERAL METHODS |
|
161
|
|
|
|
|
|
|
|
|
162
|
|
|
|
|
|
|
=head2 new(%args) |
|
163
|
|
|
|
|
|
|
|
|
164
|
|
|
|
|
|
|
Inits the object. |
|
165
|
|
|
|
|
|
|
|
|
166
|
|
|
|
|
|
|
- n_trees :: number of isolation trees in the ensemble |
|
167
|
|
|
|
|
|
|
default :: 100 |
|
168
|
|
|
|
|
|
|
|
|
169
|
|
|
|
|
|
|
- window_size :: how many of the most recent points the model reflects. |
|
170
|
|
|
|
|
|
|
Once the stream exceeds this, learning a point forgets the |
|
171
|
|
|
|
|
|
|
oldest retained point. 0 or undef disables forgetting: the |
|
172
|
|
|
|
|
|
|
model then learns from the whole stream and retains no window |
|
173
|
|
|
|
|
|
|
(so nothing is ever unlearned and threshold relearning needs |
|
174
|
|
|
|
|
|
|
caller-supplied data). |
|
175
|
|
|
|
|
|
|
default :: 2048 |
|
176
|
|
|
|
|
|
|
|
|
177
|
|
|
|
|
|
|
- max_leaf_samples :: how many points a leaf must accumulate before it |
|
178
|
|
|
|
|
|
|
splits (eta in the paper). Also the unit of the depth budget: |
|
179
|
|
|
|
|
|
|
trees stop splitting past log(n/eta)/log(4). |
|
180
|
|
|
|
|
|
|
default :: 32 |
|
181
|
|
|
|
|
|
|
|
|
182
|
|
|
|
|
|
|
- growth :: how the split requirement scales with depth (the reference |
|
183
|
|
|
|
|
|
|
implementation's `type` parameter). |
|
184
|
|
|
|
|
|
|
adaptive :: a leaf at depth k needs max_leaf_samples * 2**k |
|
185
|
|
|
|
|
|
|
points to split -- deeper splits need |
|
186
|
|
|
|
|
|
|
exponentially more evidence |
|
187
|
|
|
|
|
|
|
fixed :: max_leaf_samples points regardless of depth |
|
188
|
|
|
|
|
|
|
default :: adaptive |
|
189
|
|
|
|
|
|
|
|
|
190
|
|
|
|
|
|
|
- subsample :: probability in (0, 1] that a given tree learns (or |
|
191
|
|
|
|
|
|
|
forgets) a given point, drawn independently per tree per point. |
|
192
|
|
|
|
|
|
|
Values below 1 increase diversity among trees on very dense |
|
193
|
|
|
|
|
|
|
streams. Note that, as in the reference implementation, learn |
|
194
|
|
|
|
|
|
|
and forget draws are independent, so per-tree counts are only |
|
195
|
|
|
|
|
|
|
approximate under subsampling. |
|
196
|
|
|
|
|
|
|
default :: 1.0 |
|
197
|
|
|
|
|
|
|
|
|
198
|
|
|
|
|
|
|
- seed :: optional integer to seed srand with, for reproducible trees |
|
199
|
|
|
|
|
|
|
given the same stream in the same order. Processed via |
|
200
|
|
|
|
|
|
|
abs(int()). Seeding happens here in new(), since there is no |
|
201
|
|
|
|
|
|
|
fit() to do it in. |
|
202
|
|
|
|
|
|
|
default :: undef |
|
203
|
|
|
|
|
|
|
|
|
204
|
|
|
|
|
|
|
- contamination :: expected fraction of anomalies, in (0, 0.5]. When |
|
205
|
|
|
|
|
|
|
set, the first predict()-family call learns a score threshold |
|
206
|
|
|
|
|
|
|
that flags this fraction of the current window, and uses it as |
|
207
|
|
|
|
|
|
|
the default cutoff. The threshold does NOT track the stream |
|
208
|
|
|
|
|
|
|
automatically afterwards; call relearn_threshold() to refresh |
|
209
|
|
|
|
|
|
|
it. undef => no learned threshold (predict() falls back to |
|
210
|
|
|
|
|
|
|
0.5). |
|
211
|
|
|
|
|
|
|
default :: undef |
|
212
|
|
|
|
|
|
|
|
|
213
|
|
|
|
|
|
|
- missing :: how learn() treats undef (missing) feature cells. Scoring |
|
214
|
|
|
|
|
|
|
always tolerates undef (mapped to 0), matching the parent |
|
215
|
|
|
|
|
|
|
class's long-standing behaviour. |
|
216
|
|
|
|
|
|
|
die :: croak if a learned point contains an undef cell |
|
217
|
|
|
|
|
|
|
zero :: treat a missing cell as the value 0 |
|
218
|
|
|
|
|
|
|
default :: die |
|
219
|
|
|
|
|
|
|
|
|
220
|
|
|
|
|
|
|
- feature_names :: optional arrayref of per-feature labels enabling the |
|
221
|
|
|
|
|
|
|
*_tagged methods (and required by mungers below). |
|
222
|
|
|
|
|
|
|
default :: undef |
|
223
|
|
|
|
|
|
|
|
|
224
|
|
|
|
|
|
|
- mungers :: optional hashref of declarative L |
|
225
|
|
|
|
|
|
|
specs; every tagged row (learn_tagged, score_learn_tagged, the |
|
226
|
|
|
|
|
|
|
scoring *_tagged methods, tagged_row_to_array) is munged from |
|
227
|
|
|
|
|
|
|
raw values into numbers through the compiled plan, and |
|
228
|
|
|
|
|
|
|
munge_rows() applies the scalar mungers to positional rows. |
|
229
|
|
|
|
|
|
|
Requires feature_names; spec errors croak here. The spec is |
|
230
|
|
|
|
|
|
|
saved with the model. Identical semantics to the parent |
|
231
|
|
|
|
|
|
|
class's knob -- see MUNGERS in |
|
232
|
|
|
|
|
|
|
L for details and |
|
233
|
|
|
|
|
|
|
caveats. |
|
234
|
|
|
|
|
|
|
default :: undef |
|
235
|
|
|
|
|
|
|
|
|
236
|
|
|
|
|
|
|
- schema_version :: optional opaque string identifying the revision of |
|
237
|
|
|
|
|
|
|
the variable schema this model was built against. Never |
|
238
|
|
|
|
|
|
|
parsed; saved with the model. Usually set via a prototype -- |
|
239
|
|
|
|
|
|
|
see PROTOTYPES in L |
|
240
|
|
|
|
|
|
|
(whose new_from_prototype creates online models too). |
|
241
|
|
|
|
|
|
|
default :: undef |
|
242
|
|
|
|
|
|
|
|
|
243
|
|
|
|
|
|
|
- schema_description :: optional opaque free-text description of what |
|
244
|
|
|
|
|
|
|
the variable schema is. Same handling as schema_version. |
|
245
|
|
|
|
|
|
|
default :: undef |
|
246
|
|
|
|
|
|
|
|
|
247
|
|
|
|
|
|
|
- feature_descriptions :: optional hashref of 'feature name => free |
|
248
|
|
|
|
|
|
|
text' describing individual features. Requires feature_names; |
|
249
|
|
|
|
|
|
|
every key must name an entry there or new() croaks. Partial |
|
250
|
|
|
|
|
|
|
coverage is fine. Saved with the model. |
|
251
|
|
|
|
|
|
|
default :: undef |
|
252
|
|
|
|
|
|
|
|
|
253
|
|
|
|
|
|
|
- use_c :: boolean, override whether the parent class's Inline::C |
|
254
|
|
|
|
|
|
|
backend is used, for learning and scoring both (see |
|
255
|
|
|
|
|
|
|
DESCRIPTION). When false the instance runs pure Perl even if |
|
256
|
|
|
|
|
|
|
the C backend compiled. Results are identical either way -- |
|
257
|
|
|
|
|
|
|
learn() builds bit-identical trees for the same seed (on |
|
258
|
|
|
|
|
|
|
nvsize == 8 perls) and scoring matches exactly -- so only |
|
259
|
|
|
|
|
|
|
speed differs. |
|
260
|
|
|
|
|
|
|
default :: $Algorithm::Classifier::IsolationForest::HAS_C |
|
261
|
|
|
|
|
|
|
|
|
262
|
|
|
|
|
|
|
- use_openmp :: boolean, override whether OpenMP parallel scoring is |
|
263
|
|
|
|
|
|
|
used inside the C tree walk. Ignored when use_c is false. |
|
264
|
|
|
|
|
|
|
default :: $Algorithm::Classifier::IsolationForest::HAS_OPENMP |
|
265
|
|
|
|
|
|
|
|
|
266
|
|
|
|
|
|
|
=cut |
|
267
|
|
|
|
|
|
|
|
|
268
|
|
|
|
|
|
|
sub new { |
|
269
|
73
|
|
|
73
|
1
|
753566
|
my ( $class, %args ) = @_; |
|
270
|
|
|
|
|
|
|
|
|
271
|
73
|
|
100
|
|
|
424
|
my $growth = $args{growth} // 'adaptive'; |
|
272
|
73
|
100
|
|
|
|
713
|
croak "growth must be 'adaptive' or 'fixed'" |
|
273
|
|
|
|
|
|
|
unless $growth =~ /\A(?:adaptive|fixed)\z/; |
|
274
|
|
|
|
|
|
|
|
|
275
|
72
|
|
100
|
|
|
281
|
my $missing = $args{missing} // 'die'; |
|
276
|
72
|
100
|
|
|
|
413
|
croak "missing must be one of: die, zero" |
|
277
|
|
|
|
|
|
|
unless $missing =~ /\A(?:die|zero)\z/; |
|
278
|
|
|
|
|
|
|
|
|
279
|
71
|
100
|
|
|
|
275
|
if ( defined( $args{seed} ) ) { |
|
280
|
44
|
|
|
|
|
125
|
$args{seed} = abs( int( $args{seed} ) ); |
|
281
|
|
|
|
|
|
|
} |
|
282
|
|
|
|
|
|
|
|
|
283
|
|
|
|
|
|
|
# window_size => 0 and window_size => undef both mean "no forgetting"; |
|
284
|
|
|
|
|
|
|
# normalise to 0 so the rest of the code has one falsy spelling. |
|
285
|
71
|
100
|
100
|
|
|
304
|
my $window_size = exists $args{window_size} ? ( $args{window_size} // 0 ) : 2048; |
|
286
|
|
|
|
|
|
|
|
|
287
|
|
|
|
|
|
|
# Clamp the accel knobs against what the parent's build actually has, |
|
288
|
|
|
|
|
|
|
# exactly as the parent's new() does: use_c => 1 without a compiled |
|
289
|
|
|
|
|
|
|
# backend would otherwise call undefined XS subs at first scoring, and |
|
290
|
|
|
|
|
|
|
# OpenMP is meaningless without the C tree walk. |
|
291
|
|
|
|
|
|
|
my $use_c |
|
292
|
|
|
|
|
|
|
= defined $args{use_c} |
|
293
|
71
|
100
|
66
|
|
|
295
|
? ( $args{use_c} && $Algorithm::Classifier::IsolationForest::HAS_C ? 1 : 0 ) |
|
|
|
100
|
|
|
|
|
|
|
294
|
|
|
|
|
|
|
: $Algorithm::Classifier::IsolationForest::HAS_C; |
|
295
|
|
|
|
|
|
|
my $use_openmp |
|
296
|
|
|
|
|
|
|
= defined $args{use_openmp} |
|
297
|
71
|
50
|
33
|
|
|
185
|
? ( $args{use_openmp} && $Algorithm::Classifier::IsolationForest::HAS_OPENMP ? 1 : 0 ) |
|
|
|
100
|
|
|
|
|
|
|
298
|
|
|
|
|
|
|
: $Algorithm::Classifier::IsolationForest::HAS_OPENMP; |
|
299
|
71
|
100
|
|
|
|
220
|
$use_openmp = 0 unless $use_c; |
|
300
|
|
|
|
|
|
|
|
|
301
|
|
|
|
|
|
|
my $self = { |
|
302
|
|
|
|
|
|
|
n_trees => $args{n_trees} // 100, |
|
303
|
|
|
|
|
|
|
window_size => $window_size, |
|
304
|
|
|
|
|
|
|
max_leaf_samples => $args{max_leaf_samples} // 32, |
|
305
|
|
|
|
|
|
|
growth => $growth, |
|
306
|
|
|
|
|
|
|
subsample => $args{subsample} // 1.0, |
|
307
|
|
|
|
|
|
|
seed => $args{seed}, |
|
308
|
|
|
|
|
|
|
contamination => $args{contamination}, |
|
309
|
|
|
|
|
|
|
missing => $missing, |
|
310
|
|
|
|
|
|
|
feature_names => $args{feature_names}, |
|
311
|
|
|
|
|
|
|
threshold => undef, # learned lazily if contamination set |
|
312
|
|
|
|
|
|
|
n_features => undef, # learned from the first row |
|
313
|
|
|
|
|
|
|
seen => 0, # total points learned over the model's lifetime |
|
314
|
|
|
|
|
|
|
window => [], # the retained rows, oldest first |
|
315
|
|
|
|
|
|
|
trees => [], |
|
316
|
|
|
|
|
|
|
mungers => undef, # optional Algorithm::ToNumberMunger spec hash |
|
317
|
|
|
|
|
|
|
# Opaque schema metadata, usually set via the parent class's |
|
318
|
|
|
|
|
|
|
# new_from_prototype and persisted with the model. |
|
319
|
|
|
|
|
|
|
schema_version => $args{schema_version}, |
|
320
|
|
|
|
|
|
|
schema_description => $args{schema_description}, |
|
321
|
|
|
|
|
|
|
feature_descriptions => $args{feature_descriptions}, |
|
322
|
71
|
|
100
|
|
|
1245
|
_use_c => $use_c, |
|
|
|
|
100
|
|
|
|
|
|
|
|
|
100
|
|
|
|
|
|
323
|
|
|
|
|
|
|
_use_openmp => $use_openmp, |
|
324
|
|
|
|
|
|
|
}; |
|
325
|
|
|
|
|
|
|
|
|
326
|
71
|
|
|
|
|
183
|
for my $doc (qw(schema_version schema_description)) { |
|
327
|
|
|
|
|
|
|
croak "$doc must be a plain string" |
|
328
|
142
|
50
|
66
|
|
|
426
|
if defined $self->{$doc} && ref $self->{$doc}; |
|
329
|
|
|
|
|
|
|
} |
|
330
|
|
|
|
|
|
|
Algorithm::Classifier::IsolationForest::_validate_feature_descriptions( $self->{feature_names}, |
|
331
|
|
|
|
|
|
|
$self->{feature_descriptions} ) |
|
332
|
71
|
100
|
|
|
|
210
|
if defined $self->{feature_descriptions}; |
|
333
|
|
|
|
|
|
|
|
|
334
|
|
|
|
|
|
|
# Optional Algorithm::ToNumberMunger integration, identical to the |
|
335
|
|
|
|
|
|
|
# parent's: compiled eagerly so spec errors surface here; the module |
|
336
|
|
|
|
|
|
|
# is only required when a spec is actually given. |
|
337
|
70
|
100
|
|
|
|
175
|
if ( defined $args{mungers} ) { |
|
338
|
|
|
|
|
|
|
croak "mungers must be a hashref of 'tag => munger spec'" |
|
339
|
4
|
50
|
|
|
|
14
|
unless ref $args{mungers} eq 'HASH'; |
|
340
|
|
|
|
|
|
|
croak "mungers requires feature_names (the munger plan compiles against them)" |
|
341
|
4
|
100
|
66
|
|
|
145
|
unless ref $self->{feature_names} eq 'ARRAY' && @{ $self->{feature_names} }; |
|
|
3
|
|
|
|
|
10
|
|
|
342
|
3
|
|
|
|
|
11
|
$self->{mungers} = $args{mungers}; |
|
343
|
|
|
|
|
|
|
$self->{_munger_plan} |
|
344
|
3
|
|
|
|
|
20
|
= Algorithm::Classifier::IsolationForest::_compile_mungers( $self->{feature_names}, $self->{mungers} ); |
|
345
|
3
|
|
|
|
|
466
|
$self->{munger_module_version} = $Algorithm::ToNumberMunger::VERSION; |
|
346
|
|
|
|
|
|
|
} ## end if ( defined $args{mungers} ) |
|
347
|
|
|
|
|
|
|
|
|
348
|
69
|
100
|
|
|
|
271
|
croak "n_trees must be >= 1" unless $self->{n_trees} >= 1; |
|
349
|
68
|
100
|
|
|
|
269
|
croak "max_leaf_samples must be >= 1" unless $self->{max_leaf_samples} >= 1; |
|
350
|
|
|
|
|
|
|
croak "window_size must be 0 (unbounded) or >= max_leaf_samples" |
|
351
|
67
|
100
|
100
|
|
|
401
|
if $self->{window_size} && $self->{window_size} < $self->{max_leaf_samples}; |
|
352
|
|
|
|
|
|
|
croak "subsample must be in (0, 1]" |
|
353
|
66
|
100
|
100
|
|
|
693
|
unless $self->{subsample} > 0 && $self->{subsample} <= 1; |
|
354
|
|
|
|
|
|
|
croak "contamination must be a number in (0, 0.5]" |
|
355
|
|
|
|
|
|
|
if defined $self->{contamination} |
|
356
|
63
|
100
|
100
|
|
|
518
|
&& !( $self->{contamination} > 0 && $self->{contamination} <= 0.5 ); |
|
|
|
|
100
|
|
|
|
|
|
357
|
|
|
|
|
|
|
|
|
358
|
61
|
|
|
|
|
265
|
$self->{trees} = [ map { { root => undef, count => 0, depth_limit => 0 } } 1 .. $self->{n_trees} ]; |
|
|
2000
|
|
|
|
|
4092
|
|
|
359
|
|
|
|
|
|
|
|
|
360
|
61
|
100
|
|
|
|
302
|
srand( $self->{seed} ) if defined $self->{seed}; |
|
361
|
|
|
|
|
|
|
|
|
362
|
61
|
|
|
|
|
362
|
return bless $self, $class; |
|
363
|
|
|
|
|
|
|
} ## end sub new |
|
364
|
|
|
|
|
|
|
|
|
365
|
|
|
|
|
|
|
=head2 learn(\@data) |
|
366
|
|
|
|
|
|
|
|
|
367
|
|
|
|
|
|
|
Learns the passed samples, in order, as the next points of the stream. |
|
368
|
|
|
|
|
|
|
Once the model has seen more than C points, each learned |
|
369
|
|
|
|
|
|
|
point also forgets the oldest retained point, so the model tracks the |
|
370
|
|
|
|
|
|
|
most recent C points. |
|
371
|
|
|
|
|
|
|
|
|
372
|
|
|
|
|
|
|
The data format matches the parent class's C: an arrayref of |
|
373
|
|
|
|
|
|
|
arrayrefs, each inner arrayref one sample of numeric features. All |
|
374
|
|
|
|
|
|
|
samples must have the same feature count; the count is locked in by the |
|
375
|
|
|
|
|
|
|
first sample ever learned. |
|
376
|
|
|
|
|
|
|
|
|
377
|
|
|
|
|
|
|
Returns C<$self>, so it chains. |
|
378
|
|
|
|
|
|
|
|
|
379
|
|
|
|
|
|
|
$oif->learn(\@rows); |
|
380
|
|
|
|
|
|
|
|
|
381
|
|
|
|
|
|
|
=cut |
|
382
|
|
|
|
|
|
|
|
|
383
|
|
|
|
|
|
|
sub learn { |
|
384
|
262
|
|
|
262
|
1
|
107707
|
my ( $self, $data ) = @_; |
|
385
|
262
|
50
|
33
|
|
|
799
|
croak "learn() expects a non-empty arrayref of samples" |
|
386
|
|
|
|
|
|
|
unless ref $data eq 'ARRAY' && @$data; |
|
387
|
262
|
|
|
|
|
406
|
for my $row (@$data) { |
|
388
|
14493
|
|
|
|
|
25220
|
$self->_learn_row( $self->_prep_row( $row, 'learn' ) ); |
|
389
|
|
|
|
|
|
|
} |
|
390
|
260
|
|
|
|
|
586
|
return $self; |
|
391
|
|
|
|
|
|
|
} |
|
392
|
|
|
|
|
|
|
|
|
393
|
|
|
|
|
|
|
=head2 learn_tagged(\%row) |
|
394
|
|
|
|
|
|
|
|
|
395
|
|
|
|
|
|
|
=head2 learn_tagged(\@rows) |
|
396
|
|
|
|
|
|
|
|
|
397
|
|
|
|
|
|
|
Learns one sample supplied as a hashref of named feature values, or a |
|
398
|
|
|
|
|
|
|
whole batch supplied as an arrayref of such hashrefs, in stream order. |
|
399
|
|
|
|
|
|
|
The model must have C set. Rows go through |
|
400
|
|
|
|
|
|
|
L (and therefore through the munger plan when |
|
401
|
|
|
|
|
|
|
C is configured). Returns C<$self>. |
|
402
|
|
|
|
|
|
|
|
|
403
|
|
|
|
|
|
|
$oif->learn_tagged({ cpu => 0.9, mem => 0.4, disk => 0.1 }); |
|
404
|
|
|
|
|
|
|
$oif->learn_tagged(\@hashref_rows); |
|
405
|
|
|
|
|
|
|
|
|
406
|
|
|
|
|
|
|
Croaks under the same conditions as L, naming the |
|
407
|
|
|
|
|
|
|
offending row by index in the batch form. |
|
408
|
|
|
|
|
|
|
|
|
409
|
|
|
|
|
|
|
=cut |
|
410
|
|
|
|
|
|
|
|
|
411
|
|
|
|
|
|
|
sub learn_tagged { |
|
412
|
3
|
|
|
3
|
1
|
442
|
my ( $self, $row ) = @_; |
|
413
|
3
|
100
|
|
|
|
13
|
if ( ref $row eq 'ARRAY' ) { |
|
414
|
1
|
|
|
|
|
2
|
my @rows; |
|
415
|
1
|
|
|
|
|
4
|
for my $i ( 0 .. $#$row ) { |
|
416
|
200
|
|
|
|
|
360
|
push @rows, $self->tagged_row_to_array( $row->[$i], "learn_tagged (row $i)" ); |
|
417
|
|
|
|
|
|
|
} |
|
418
|
1
|
|
|
|
|
6
|
return $self->learn( \@rows ); |
|
419
|
|
|
|
|
|
|
} |
|
420
|
2
|
|
|
|
|
10
|
my $vec = $self->tagged_row_to_array( $row, 'learn_tagged' ); |
|
421
|
2
|
|
|
|
|
8
|
return $self->learn( [$vec] ); |
|
422
|
|
|
|
|
|
|
} ## end sub learn_tagged |
|
423
|
|
|
|
|
|
|
|
|
424
|
|
|
|
|
|
|
=head2 score_learn(\@data) |
|
425
|
|
|
|
|
|
|
|
|
426
|
|
|
|
|
|
|
Prequential (test-then-train) operation, the usual way to run a streaming |
|
427
|
|
|
|
|
|
|
detector: each sample is scored against the model as it stood I |
|
428
|
|
|
|
|
|
|
that sample was learned, then learned. Returns an arrayref of anomaly |
|
429
|
|
|
|
|
|
|
scores, one per sample, in input order. |
|
430
|
|
|
|
|
|
|
|
|
431
|
|
|
|
|
|
|
Unlike the pure scoring methods this works on a brand-new model too (the |
|
432
|
|
|
|
|
|
|
first points of a stream simply score 1.0, as nothing is known yet). |
|
433
|
|
|
|
|
|
|
|
|
434
|
|
|
|
|
|
|
my $scores = $oif->score_learn(\@rows); |
|
435
|
|
|
|
|
|
|
|
|
436
|
|
|
|
|
|
|
=cut |
|
437
|
|
|
|
|
|
|
|
|
438
|
|
|
|
|
|
|
sub score_learn { |
|
439
|
35
|
|
|
35
|
1
|
8599
|
my ( $self, $data ) = @_; |
|
440
|
35
|
50
|
33
|
|
|
233
|
croak "score_learn() expects a non-empty arrayref of samples" |
|
441
|
|
|
|
|
|
|
unless ref $data eq 'ARRAY' && @$data; |
|
442
|
35
|
|
|
|
|
70
|
my @scores; |
|
443
|
35
|
|
|
|
|
91
|
for my $row (@$data) { |
|
444
|
1925
|
|
|
|
|
3528
|
my $r = $self->_prep_row( $row, 'score_learn' ); |
|
445
|
1924
|
|
|
|
|
3577
|
push @scores, $self->_score_row($r); |
|
446
|
1924
|
|
|
|
|
3087
|
$self->_learn_row($r); |
|
447
|
|
|
|
|
|
|
} |
|
448
|
34
|
|
|
|
|
485
|
return \@scores; |
|
449
|
|
|
|
|
|
|
} ## end sub score_learn |
|
450
|
|
|
|
|
|
|
|
|
451
|
|
|
|
|
|
|
=head2 score_learn_tagged(\%row) |
|
452
|
|
|
|
|
|
|
|
|
453
|
|
|
|
|
|
|
Prequential score-then-learn for a single sample supplied as a hashref of |
|
454
|
|
|
|
|
|
|
named feature values. Returns the scalar anomaly score the sample had |
|
455
|
|
|
|
|
|
|
before it was learned. |
|
456
|
|
|
|
|
|
|
|
|
457
|
|
|
|
|
|
|
my $score = $oif->score_learn_tagged({ cpu => 0.9, mem => 0.4 }); |
|
458
|
|
|
|
|
|
|
|
|
459
|
|
|
|
|
|
|
Croaks under the same conditions as L. |
|
460
|
|
|
|
|
|
|
|
|
461
|
|
|
|
|
|
|
=cut |
|
462
|
|
|
|
|
|
|
|
|
463
|
|
|
|
|
|
|
sub score_learn_tagged { |
|
464
|
2
|
|
|
2
|
1
|
1580
|
my ( $self, $row ) = @_; |
|
465
|
2
|
|
|
|
|
14
|
my $vec = $self->tagged_row_to_array( $row, 'score_learn_tagged' ); |
|
466
|
2
|
|
|
|
|
15
|
my $result = $self->score_learn( [$vec] ); |
|
467
|
2
|
|
|
|
|
16
|
return $result->[0]; |
|
468
|
|
|
|
|
|
|
} |
|
469
|
|
|
|
|
|
|
|
|
470
|
|
|
|
|
|
|
=head2 score_samples(\@data) |
|
471
|
|
|
|
|
|
|
|
|
472
|
|
|
|
|
|
|
Returns an arrayref of anomaly scores in (0, 1] without learning |
|
473
|
|
|
|
|
|
|
anything. Scores near 1 are strong anomalies (isolated at shallow |
|
474
|
|
|
|
|
|
|
depth); scores well below 0.5 are normal. |
|
475
|
|
|
|
|
|
|
|
|
476
|
|
|
|
|
|
|
my $scores = $oif->score_samples(\@data); |
|
477
|
|
|
|
|
|
|
|
|
478
|
|
|
|
|
|
|
=cut |
|
479
|
|
|
|
|
|
|
|
|
480
|
|
|
|
|
|
|
sub score_samples { |
|
481
|
69
|
|
|
69
|
1
|
51885
|
my ( $self, $data ) = @_; |
|
482
|
69
|
|
|
|
|
292
|
$self->_check_learned; |
|
483
|
68
|
50
|
|
|
|
203
|
croak "score_samples() expects an arrayref of samples" |
|
484
|
|
|
|
|
|
|
unless ref $data eq 'ARRAY'; |
|
485
|
|
|
|
|
|
|
|
|
486
|
68
|
100
|
|
|
|
226
|
if ( $self->_ensure_c_trees ) { |
|
487
|
55
|
|
|
|
|
188
|
my ( $n_pts, $x_packed ) = $self->_pack_input($data); |
|
488
|
55
|
|
|
|
|
202
|
my $sums_packed = "\0" x ( $n_pts * 8 ); |
|
489
|
|
|
|
|
|
|
Algorithm::Classifier::IsolationForest::score_all_xs( |
|
490
|
|
|
|
|
|
|
$self->{_c_nodes}, $self->{_c_coef_idx}, $self->{_c_coef_val}, |
|
491
|
|
|
|
|
|
|
$x_packed, $sums_packed, $n_pts, |
|
492
|
55
|
|
|
|
|
256518
|
$self->{n_features}, scalar @{ $self->{trees} }, $self->{_use_openmp} |
|
493
|
55
|
|
|
|
|
186
|
); |
|
494
|
55
|
|
|
|
|
311
|
my $result = []; |
|
495
|
55
|
|
|
|
|
339
|
Algorithm::Classifier::IsolationForest::finalize_scores_xs( $sums_packed, $n_pts, $self->_score_inv, $result ); |
|
496
|
55
|
|
|
|
|
2824
|
return $result; |
|
497
|
|
|
|
|
|
|
} ## end if ( $self->_ensure_c_trees ) |
|
498
|
|
|
|
|
|
|
|
|
499
|
13
|
|
|
|
|
33
|
my $sums = $self->_depth_sums($data); |
|
500
|
13
|
|
|
|
|
31
|
my $inv = $self->_score_inv; |
|
501
|
13
|
|
|
|
|
32
|
return [ map { exp( -$_ * $inv ) } @$sums ]; |
|
|
567
|
|
|
|
|
779
|
|
|
502
|
|
|
|
|
|
|
} ## end sub score_samples |
|
503
|
|
|
|
|
|
|
|
|
504
|
|
|
|
|
|
|
=head2 score_sample_tagged(\%row) |
|
505
|
|
|
|
|
|
|
|
|
506
|
|
|
|
|
|
|
Scores a single sample supplied as a hashref of named feature values, |
|
507
|
|
|
|
|
|
|
without learning it. Returns a scalar anomaly score in (0, 1]. |
|
508
|
|
|
|
|
|
|
|
|
509
|
|
|
|
|
|
|
my $score = $oif->score_sample_tagged({ cpu => 0.9, mem => 0.4 }); |
|
510
|
|
|
|
|
|
|
|
|
511
|
|
|
|
|
|
|
Croaks under the same conditions as L. |
|
512
|
|
|
|
|
|
|
|
|
513
|
|
|
|
|
|
|
=cut |
|
514
|
|
|
|
|
|
|
|
|
515
|
|
|
|
|
|
|
sub score_sample_tagged { |
|
516
|
7
|
|
|
7
|
1
|
2171
|
my ( $self, $row ) = @_; |
|
517
|
7
|
|
|
|
|
24
|
my $vec = $self->tagged_row_to_array( $row, 'score_sample_tagged' ); |
|
518
|
4
|
|
|
|
|
19
|
my $result = $self->score_samples( [$vec] ); |
|
519
|
4
|
|
|
|
|
39
|
return $result->[0]; |
|
520
|
|
|
|
|
|
|
} |
|
521
|
|
|
|
|
|
|
|
|
522
|
|
|
|
|
|
|
=head2 path_lengths(\@data) |
|
523
|
|
|
|
|
|
|
|
|
524
|
|
|
|
|
|
|
Returns an arrayref of the mean isolation depth per sample across the |
|
525
|
|
|
|
|
|
|
trees, for inspection -- the streaming counterpart of the parent class's |
|
526
|
|
|
|
|
|
|
method of the same name. Depths include the per-leaf count adjustment. |
|
527
|
|
|
|
|
|
|
|
|
528
|
|
|
|
|
|
|
my $depths = $oif->path_lengths(\@data); |
|
529
|
|
|
|
|
|
|
|
|
530
|
|
|
|
|
|
|
=cut |
|
531
|
|
|
|
|
|
|
|
|
532
|
|
|
|
|
|
|
sub path_lengths { |
|
533
|
4
|
|
|
4
|
1
|
3608
|
my ( $self, $data ) = @_; |
|
534
|
4
|
|
|
|
|
14
|
$self->_check_learned; |
|
535
|
3
|
50
|
|
|
|
10
|
croak "path_lengths() expects an arrayref of samples" |
|
536
|
|
|
|
|
|
|
unless ref $data eq 'ARRAY'; |
|
537
|
3
|
|
|
|
|
6
|
my $t = scalar @{ $self->{trees} }; |
|
|
3
|
|
|
|
|
6
|
|
|
538
|
|
|
|
|
|
|
|
|
539
|
3
|
100
|
|
|
|
71
|
if ( $self->_ensure_c_trees ) { |
|
540
|
2
|
|
|
|
|
6
|
my ( $n_pts, $x_packed ) = $self->_pack_input($data); |
|
541
|
2
|
|
|
|
|
7
|
my $sums_packed = "\0" x ( $n_pts * 8 ); |
|
542
|
|
|
|
|
|
|
Algorithm::Classifier::IsolationForest::score_all_xs( |
|
543
|
|
|
|
|
|
|
$self->{_c_nodes}, $self->{_c_coef_idx}, $self->{_c_coef_val}, |
|
544
|
|
|
|
|
|
|
$x_packed, $sums_packed, $n_pts, |
|
545
|
|
|
|
|
|
|
$self->{n_features}, $t, $self->{_use_openmp} |
|
546
|
2
|
|
|
|
|
2915
|
); |
|
547
|
2
|
|
|
|
|
7
|
my $result = []; |
|
548
|
2
|
|
|
|
|
14
|
Algorithm::Classifier::IsolationForest::finalize_path_lengths_xs( $sums_packed, $n_pts, $t + 0.0, $result ); |
|
549
|
2
|
|
|
|
|
11
|
return $result; |
|
550
|
|
|
|
|
|
|
} ## end if ( $self->_ensure_c_trees ) |
|
551
|
|
|
|
|
|
|
|
|
552
|
1
|
|
|
|
|
3
|
my $sums = $self->_depth_sums($data); |
|
553
|
1
|
|
|
|
|
3
|
return [ map { $_ / $t } @$sums ]; |
|
|
5
|
|
|
|
|
12
|
|
|
554
|
|
|
|
|
|
|
} ## end sub path_lengths |
|
555
|
|
|
|
|
|
|
|
|
556
|
|
|
|
|
|
|
=head2 predict(\@data, $threshold) |
|
557
|
|
|
|
|
|
|
|
|
558
|
|
|
|
|
|
|
Returns an arrayref of 0/1 labels for the specified data, without |
|
559
|
|
|
|
|
|
|
learning it. |
|
560
|
|
|
|
|
|
|
|
|
561
|
|
|
|
|
|
|
If C<$threshold> is not given, the contamination-learned cutoff is used |
|
562
|
|
|
|
|
|
|
when available (learned from the current window on first use -- see |
|
563
|
|
|
|
|
|
|
C in L), otherwise 0.5. |
|
564
|
|
|
|
|
|
|
|
|
565
|
|
|
|
|
|
|
Note that absolute score levels depend on C and |
|
566
|
|
|
|
|
|
|
C (shallower depth budgets compress scores downward), |
|
567
|
|
|
|
|
|
|
so the 0.5 fallback is a blunt default here -- anomalies reliably rank |
|
568
|
|
|
|
|
|
|
above normal points, but may sit below 0.5. Setting C, |
|
569
|
|
|
|
|
|
|
or passing a threshold calibrated from observed scores, is recommended. |
|
570
|
|
|
|
|
|
|
|
|
571
|
|
|
|
|
|
|
my $labels = $oif->predict(\@data); |
|
572
|
|
|
|
|
|
|
|
|
573
|
|
|
|
|
|
|
=cut |
|
574
|
|
|
|
|
|
|
|
|
575
|
|
|
|
|
|
|
sub predict { |
|
576
|
17
|
|
|
17
|
1
|
2314
|
my ( $self, $data, $threshold ) = @_; |
|
577
|
17
|
|
|
|
|
60
|
$self->_check_learned; |
|
578
|
16
|
|
|
|
|
106
|
$self->_ensure_threshold; |
|
579
|
|
|
|
|
|
|
$threshold |
|
580
|
|
|
|
|
|
|
= defined $threshold ? $threshold |
|
581
|
|
|
|
|
|
|
: defined $self->{threshold} ? $self->{threshold} |
|
582
|
16
|
50
|
|
|
|
59
|
: 0.5; |
|
|
|
100
|
|
|
|
|
|
|
583
|
|
|
|
|
|
|
|
|
584
|
|
|
|
|
|
|
# Fast path: threshold the raw depth sums directly, skipping the |
|
585
|
|
|
|
|
|
|
# per-point exp() -- score >= T iff sum <= -log(T)/inv. Only valid |
|
586
|
|
|
|
|
|
|
# for a normal threshold in (0, 1), like the parent's gate. |
|
587
|
16
|
100
|
66
|
|
|
91
|
if ( $threshold > 0 && $threshold < 1 && $self->_ensure_c_trees ) { |
|
|
|
|
100
|
|
|
|
|
|
588
|
9
|
|
|
|
|
27
|
my ( $n_pts, $x_packed ) = $self->_pack_input($data); |
|
589
|
9
|
|
|
|
|
33
|
my $sums_packed = "\0" x ( $n_pts * 8 ); |
|
590
|
|
|
|
|
|
|
Algorithm::Classifier::IsolationForest::score_all_xs( |
|
591
|
|
|
|
|
|
|
$self->{_c_nodes}, $self->{_c_coef_idx}, $self->{_c_coef_val}, |
|
592
|
|
|
|
|
|
|
$x_packed, $sums_packed, $n_pts, |
|
593
|
9
|
|
|
|
|
25770
|
$self->{n_features}, scalar @{ $self->{trees} }, $self->{_use_openmp} |
|
594
|
9
|
|
|
|
|
28
|
); |
|
595
|
9
|
|
|
|
|
77
|
my $sum_threshold = -log($threshold) / $self->_score_inv; |
|
596
|
9
|
|
|
|
|
22
|
my $result = []; |
|
597
|
9
|
|
|
|
|
74
|
Algorithm::Classifier::IsolationForest::predict_sums_xs( $sums_packed, $n_pts, $sum_threshold, $result ); |
|
598
|
9
|
|
|
|
|
45
|
return $result; |
|
599
|
|
|
|
|
|
|
} ## end if ( $threshold > 0 && $threshold < 1 && $self...) |
|
600
|
|
|
|
|
|
|
|
|
601
|
7
|
|
|
|
|
19
|
my $scores = $self->score_samples($data); |
|
602
|
7
|
100
|
|
|
|
12
|
return [ map { $_ >= $threshold ? 1 : 0 } @$scores ]; |
|
|
35
|
|
|
|
|
84
|
|
|
603
|
|
|
|
|
|
|
} ## end sub predict |
|
604
|
|
|
|
|
|
|
|
|
605
|
|
|
|
|
|
|
=head2 predict_tagged(\%row, $threshold) |
|
606
|
|
|
|
|
|
|
|
|
607
|
|
|
|
|
|
|
Predicts whether a single sample, supplied as a hashref of named feature |
|
608
|
|
|
|
|
|
|
values, is an anomaly. Returns a scalar 1 (anomaly) or 0 (normal). |
|
609
|
|
|
|
|
|
|
C<$threshold> defaults the same way as in L. |
|
610
|
|
|
|
|
|
|
|
|
611
|
|
|
|
|
|
|
my $label = $oif->predict_tagged({ cpu => 0.9, mem => 0.4 }); |
|
612
|
|
|
|
|
|
|
|
|
613
|
|
|
|
|
|
|
Croaks under the same conditions as L. |
|
614
|
|
|
|
|
|
|
|
|
615
|
|
|
|
|
|
|
=cut |
|
616
|
|
|
|
|
|
|
|
|
617
|
|
|
|
|
|
|
sub predict_tagged { |
|
618
|
1
|
|
|
1
|
1
|
1245
|
my ( $self, $row, $threshold ) = @_; |
|
619
|
1
|
|
|
|
|
8
|
my $vec = $self->tagged_row_to_array( $row, 'predict_tagged' ); |
|
620
|
1
|
|
|
|
|
9
|
my $result = $self->predict( [$vec], $threshold ); |
|
621
|
1
|
|
|
|
|
10
|
return $result->[0]; |
|
622
|
|
|
|
|
|
|
} |
|
623
|
|
|
|
|
|
|
|
|
624
|
|
|
|
|
|
|
=head2 score_predict_samples(\@data, $threshold) |
|
625
|
|
|
|
|
|
|
|
|
626
|
|
|
|
|
|
|
Returns an arrayref of C<[$score, $label]> pairs, one per sample, without |
|
627
|
|
|
|
|
|
|
learning. C<$threshold> defaults the same way as in L. |
|
628
|
|
|
|
|
|
|
|
|
629
|
|
|
|
|
|
|
my $results = $oif->score_predict_samples(\@data); |
|
630
|
|
|
|
|
|
|
|
|
631
|
|
|
|
|
|
|
=cut |
|
632
|
|
|
|
|
|
|
|
|
633
|
|
|
|
|
|
|
sub score_predict_samples { |
|
634
|
4
|
|
|
4
|
1
|
872
|
my ( $self, $data, $threshold ) = @_; |
|
635
|
4
|
|
|
|
|
16
|
$self->_check_learned; |
|
636
|
4
|
|
|
|
|
15
|
$self->_ensure_threshold; |
|
637
|
|
|
|
|
|
|
$threshold |
|
638
|
|
|
|
|
|
|
= defined $threshold ? $threshold |
|
639
|
|
|
|
|
|
|
: defined $self->{threshold} ? $self->{threshold} |
|
640
|
4
|
50
|
|
|
|
15
|
: 0.5; |
|
|
|
100
|
|
|
|
|
|
|
641
|
|
|
|
|
|
|
|
|
642
|
|
|
|
|
|
|
# Fast path: [score, label] pairs built straight from the sum buffer |
|
643
|
|
|
|
|
|
|
# in one C call; gated identically to predict(). |
|
644
|
4
|
100
|
33
|
|
|
35
|
if ( $threshold > 0 && $threshold < 1 && $self->_ensure_c_trees ) { |
|
|
|
|
66
|
|
|
|
|
|
645
|
3
|
|
|
|
|
12
|
my ( $n_pts, $x_packed ) = $self->_pack_input($data); |
|
646
|
3
|
|
|
|
|
10
|
my $sums_packed = "\0" x ( $n_pts * 8 ); |
|
647
|
|
|
|
|
|
|
Algorithm::Classifier::IsolationForest::score_all_xs( |
|
648
|
|
|
|
|
|
|
$self->{_c_nodes}, $self->{_c_coef_idx}, $self->{_c_coef_val}, |
|
649
|
|
|
|
|
|
|
$x_packed, $sums_packed, $n_pts, |
|
650
|
3
|
|
|
|
|
27429
|
$self->{n_features}, scalar @{ $self->{trees} }, $self->{_use_openmp} |
|
651
|
3
|
|
|
|
|
12
|
); |
|
652
|
3
|
|
|
|
|
44
|
my $inv = $self->_score_inv; |
|
653
|
3
|
|
|
|
|
10
|
my $sum_threshold = -log($threshold) / $inv; |
|
654
|
3
|
|
|
|
|
8
|
my $result = []; |
|
655
|
3
|
|
|
|
|
72
|
Algorithm::Classifier::IsolationForest::score_predict_xs( $sums_packed, $n_pts, $inv, $sum_threshold, $result ); |
|
656
|
3
|
|
|
|
|
21
|
return $result; |
|
657
|
|
|
|
|
|
|
} ## end if ( $threshold > 0 && $threshold < 1 && $self...) |
|
658
|
|
|
|
|
|
|
|
|
659
|
1
|
|
|
|
|
3
|
my $scores = $self->score_samples($data); |
|
660
|
1
|
100
|
|
|
|
4
|
return [ map { [ $_, ( $_ >= $threshold ? 1 : 0 ) ] } @$scores ]; |
|
|
5
|
|
|
|
|
24
|
|
|
661
|
|
|
|
|
|
|
} ## end sub score_predict_samples |
|
662
|
|
|
|
|
|
|
|
|
663
|
|
|
|
|
|
|
=head2 score_predict_sample_tagged(\%row, $threshold) |
|
664
|
|
|
|
|
|
|
|
|
665
|
|
|
|
|
|
|
Scores and classifies a single sample supplied as a hashref of named |
|
666
|
|
|
|
|
|
|
feature values. Returns a two-element arrayref C<[$score, $label]>. |
|
667
|
|
|
|
|
|
|
C<$threshold> defaults the same way as in L. |
|
668
|
|
|
|
|
|
|
|
|
669
|
|
|
|
|
|
|
my $pair = $oif->score_predict_sample_tagged({ cpu => 0.9, mem => 0.4 }); |
|
670
|
|
|
|
|
|
|
|
|
671
|
|
|
|
|
|
|
Croaks under the same conditions as L. |
|
672
|
|
|
|
|
|
|
|
|
673
|
|
|
|
|
|
|
=cut |
|
674
|
|
|
|
|
|
|
|
|
675
|
|
|
|
|
|
|
sub score_predict_sample_tagged { |
|
676
|
1
|
|
|
1
|
1
|
1737
|
my ( $self, $row, $threshold ) = @_; |
|
677
|
1
|
|
|
|
|
9
|
my $vec = $self->tagged_row_to_array( $row, 'score_predict_sample_tagged' ); |
|
678
|
1
|
|
|
|
|
11
|
my $result = $self->score_predict_samples( [$vec], $threshold ); |
|
679
|
1
|
|
|
|
|
25
|
return $result->[0]; |
|
680
|
|
|
|
|
|
|
} |
|
681
|
|
|
|
|
|
|
|
|
682
|
|
|
|
|
|
|
=head2 score_predict_split(\@data, $threshold) |
|
683
|
|
|
|
|
|
|
|
|
684
|
|
|
|
|
|
|
Same values as L but returned as two flat |
|
685
|
|
|
|
|
|
|
arrayrefs. In list context returns C<($scores_aref, $labels_aref)>. |
|
686
|
|
|
|
|
|
|
|
|
687
|
|
|
|
|
|
|
my ($scores, $labels) = $oif->score_predict_split(\@data); |
|
688
|
|
|
|
|
|
|
|
|
689
|
|
|
|
|
|
|
=cut |
|
690
|
|
|
|
|
|
|
|
|
691
|
|
|
|
|
|
|
sub score_predict_split { |
|
692
|
3
|
|
|
3
|
1
|
1053
|
my ( $self, $data, $threshold ) = @_; |
|
693
|
3
|
|
|
|
|
11
|
$self->_check_learned; |
|
694
|
3
|
|
|
|
|
10
|
$self->_ensure_threshold; |
|
695
|
|
|
|
|
|
|
$threshold |
|
696
|
|
|
|
|
|
|
= defined $threshold ? $threshold |
|
697
|
|
|
|
|
|
|
: defined $self->{threshold} ? $self->{threshold} |
|
698
|
3
|
50
|
|
|
|
11
|
: 0.5; |
|
|
|
100
|
|
|
|
|
|
|
699
|
|
|
|
|
|
|
|
|
700
|
|
|
|
|
|
|
# Fast path: two flat arrayrefs straight from the sum buffer; gated |
|
701
|
|
|
|
|
|
|
# identically to predict(). |
|
702
|
3
|
100
|
33
|
|
|
27
|
if ( $threshold > 0 && $threshold < 1 && $self->_ensure_c_trees ) { |
|
|
|
|
66
|
|
|
|
|
|
703
|
2
|
|
|
|
|
9
|
my ( $n_pts, $x_packed ) = $self->_pack_input($data); |
|
704
|
2
|
|
|
|
|
8
|
my $sums_packed = "\0" x ( $n_pts * 8 ); |
|
705
|
|
|
|
|
|
|
Algorithm::Classifier::IsolationForest::score_all_xs( |
|
706
|
|
|
|
|
|
|
$self->{_c_nodes}, $self->{_c_coef_idx}, $self->{_c_coef_val}, |
|
707
|
|
|
|
|
|
|
$x_packed, $sums_packed, $n_pts, |
|
708
|
2
|
|
|
|
|
5305
|
$self->{n_features}, scalar @{ $self->{trees} }, $self->{_use_openmp} |
|
709
|
2
|
|
|
|
|
9
|
); |
|
710
|
2
|
|
|
|
|
13
|
my $inv = $self->_score_inv; |
|
711
|
2
|
|
|
|
|
6
|
my $sum_threshold = -log($threshold) / $inv; |
|
712
|
2
|
|
|
|
|
20
|
my $scores = []; |
|
713
|
2
|
|
|
|
|
5
|
my $labels = []; |
|
714
|
2
|
|
|
|
|
14
|
Algorithm::Classifier::IsolationForest::score_predict_split_xs( $sums_packed, $n_pts, $inv, |
|
715
|
|
|
|
|
|
|
$sum_threshold, $scores, $labels ); |
|
716
|
2
|
|
|
|
|
13
|
return ( $scores, $labels ); |
|
717
|
|
|
|
|
|
|
} ## end if ( $threshold > 0 && $threshold < 1 && $self...) |
|
718
|
|
|
|
|
|
|
|
|
719
|
1
|
|
|
|
|
2
|
my $scores = $self->score_samples($data); |
|
720
|
1
|
100
|
|
|
|
2
|
my @labels = map { $_ >= $threshold ? 1 : 0 } @$scores; |
|
|
5
|
|
|
|
|
9
|
|
|
721
|
1
|
|
|
|
|
4
|
return ( $scores, \@labels ); |
|
722
|
|
|
|
|
|
|
} ## end sub score_predict_split |
|
723
|
|
|
|
|
|
|
|
|
724
|
|
|
|
|
|
|
=head2 relearn_threshold(\@data) |
|
725
|
|
|
|
|
|
|
|
|
726
|
|
|
|
|
|
|
Re-derives the contamination decision threshold so it flags the requested |
|
727
|
|
|
|
|
|
|
fraction of the current window (or of C<\@data>, when passed). Call this |
|
728
|
|
|
|
|
|
|
after the stream has drifted, or on whatever cadence threshold freshness |
|
729
|
|
|
|
|
|
|
matters; learning alone never moves the threshold. |
|
730
|
|
|
|
|
|
|
|
|
731
|
|
|
|
|
|
|
Requires C to have been set. With C<< window_size => 0 >> |
|
732
|
|
|
|
|
|
|
no window is retained, so C<\@data> must be supplied. |
|
733
|
|
|
|
|
|
|
|
|
734
|
|
|
|
|
|
|
Returns C<$self>, so it chains. |
|
735
|
|
|
|
|
|
|
|
|
736
|
|
|
|
|
|
|
$oif->relearn_threshold; |
|
737
|
|
|
|
|
|
|
|
|
738
|
|
|
|
|
|
|
=cut |
|
739
|
|
|
|
|
|
|
|
|
740
|
|
|
|
|
|
|
sub relearn_threshold { |
|
741
|
9
|
|
|
9
|
1
|
99
|
my ( $self, $data ) = @_; |
|
742
|
|
|
|
|
|
|
croak "relearn_threshold requires contamination to have been set in new()" |
|
743
|
9
|
100
|
|
|
|
454
|
unless defined $self->{contamination}; |
|
744
|
7
|
100
|
|
|
|
26
|
my $rows = defined $data ? $data : $self->{window}; |
|
745
|
7
|
100
|
66
|
|
|
210
|
croak "relearn_threshold: no retained window to learn a threshold from " |
|
746
|
|
|
|
|
|
|
. "(window_size is 0); pass an arrayref of recent data" |
|
747
|
|
|
|
|
|
|
unless ref $rows eq 'ARRAY' && @$rows; |
|
748
|
|
|
|
|
|
|
|
|
749
|
6
|
|
|
|
|
29
|
my $scores = $self->score_samples($rows); |
|
750
|
6
|
|
|
|
|
95
|
my @desc = sort { $b <=> $a } @$scores; |
|
|
13458
|
|
|
|
|
16606
|
|
|
751
|
6
|
|
|
|
|
17
|
my $n_pts = scalar @desc; |
|
752
|
6
|
|
|
|
|
27
|
my $k = int( $self->{contamination} * $n_pts + 0.5 ); |
|
753
|
6
|
50
|
|
|
|
33
|
$k = 1 if $k < 1; |
|
754
|
6
|
50
|
|
|
|
58
|
$k = $n_pts if $k > $n_pts; |
|
755
|
6
|
|
|
|
|
49
|
$self->{threshold} = Algorithm::Classifier::IsolationForest::_threshold_from_ranked( \@desc, $k ); |
|
756
|
6
|
|
|
|
|
92
|
return $self; |
|
757
|
|
|
|
|
|
|
} ## end sub relearn_threshold |
|
758
|
|
|
|
|
|
|
|
|
759
|
|
|
|
|
|
|
=head2 decision_threshold |
|
760
|
|
|
|
|
|
|
|
|
761
|
|
|
|
|
|
|
The score cutoff the predict methods use by default; undef unless |
|
762
|
|
|
|
|
|
|
C was set and a predict-family method or |
|
763
|
|
|
|
|
|
|
L has run. |
|
764
|
|
|
|
|
|
|
|
|
765
|
|
|
|
|
|
|
=cut |
|
766
|
|
|
|
|
|
|
|
|
767
|
43
|
|
|
43
|
1
|
3413
|
sub decision_threshold { return $_[0]->{threshold} } |
|
768
|
|
|
|
|
|
|
|
|
769
|
|
|
|
|
|
|
=head2 feature_names |
|
770
|
|
|
|
|
|
|
|
|
771
|
|
|
|
|
|
|
Returns the arrayref of feature name strings stored with the model, or |
|
772
|
|
|
|
|
|
|
undef if none were provided. |
|
773
|
|
|
|
|
|
|
|
|
774
|
|
|
|
|
|
|
=cut |
|
775
|
|
|
|
|
|
|
|
|
776
|
2
|
|
|
2
|
1
|
12
|
sub feature_names { return $_[0]->{feature_names} } |
|
777
|
|
|
|
|
|
|
|
|
778
|
|
|
|
|
|
|
=head2 schema_version |
|
779
|
|
|
|
|
|
|
|
|
780
|
|
|
|
|
|
|
Returns the user-owned schema version string stored with the model |
|
781
|
|
|
|
|
|
|
(usually via a prototype -- see PROTOTYPES in |
|
782
|
|
|
|
|
|
|
L), or undef if none was |
|
783
|
|
|
|
|
|
|
recorded. |
|
784
|
|
|
|
|
|
|
|
|
785
|
|
|
|
|
|
|
=cut |
|
786
|
|
|
|
|
|
|
|
|
787
|
2
|
|
|
2
|
1
|
1014
|
sub schema_version { return $_[0]->{schema_version} } |
|
788
|
|
|
|
|
|
|
|
|
789
|
|
|
|
|
|
|
=head2 schema_description |
|
790
|
|
|
|
|
|
|
|
|
791
|
|
|
|
|
|
|
Returns the free-text description of the variable schema stored with the |
|
792
|
|
|
|
|
|
|
model, or undef if none was recorded. |
|
793
|
|
|
|
|
|
|
|
|
794
|
|
|
|
|
|
|
=cut |
|
795
|
|
|
|
|
|
|
|
|
796
|
0
|
|
|
0
|
1
|
0
|
sub schema_description { return $_[0]->{schema_description} } |
|
797
|
|
|
|
|
|
|
|
|
798
|
|
|
|
|
|
|
=head2 feature_descriptions |
|
799
|
|
|
|
|
|
|
|
|
800
|
|
|
|
|
|
|
Returns the hashref of per-feature description strings stored with the |
|
801
|
|
|
|
|
|
|
model, or undef if none were recorded. Keys are feature names; coverage |
|
802
|
|
|
|
|
|
|
may be partial. |
|
803
|
|
|
|
|
|
|
|
|
804
|
|
|
|
|
|
|
=cut |
|
805
|
|
|
|
|
|
|
|
|
806
|
3
|
|
|
3
|
1
|
3737
|
sub feature_descriptions { return $_[0]->{feature_descriptions} } |
|
807
|
|
|
|
|
|
|
|
|
808
|
|
|
|
|
|
|
=head2 window_count |
|
809
|
|
|
|
|
|
|
|
|
810
|
|
|
|
|
|
|
Returns how many points the model currently retains in its sliding |
|
811
|
|
|
|
|
|
|
window (0 when C<< window_size => 0 >>). |
|
812
|
|
|
|
|
|
|
|
|
813
|
|
|
|
|
|
|
=cut |
|
814
|
|
|
|
|
|
|
|
|
815
|
22
|
|
|
22
|
1
|
16817
|
sub window_count { return scalar @{ $_[0]->{window} } } |
|
|
22
|
|
|
|
|
186
|
|
|
816
|
|
|
|
|
|
|
|
|
817
|
|
|
|
|
|
|
=head2 seen |
|
818
|
|
|
|
|
|
|
|
|
819
|
|
|
|
|
|
|
Returns the total number of points learned over the model's lifetime, |
|
820
|
|
|
|
|
|
|
including points that have since been forgotten. |
|
821
|
|
|
|
|
|
|
|
|
822
|
|
|
|
|
|
|
=cut |
|
823
|
|
|
|
|
|
|
|
|
824
|
36
|
|
|
36
|
1
|
5026
|
sub seen { return $_[0]->{seen} } |
|
825
|
|
|
|
|
|
|
|
|
826
|
|
|
|
|
|
|
=head2 tagged_row_to_array(\%row, $caller) |
|
827
|
|
|
|
|
|
|
|
|
828
|
|
|
|
|
|
|
Validates a hashref of named feature values against the model's stored |
|
829
|
|
|
|
|
|
|
C and returns a positional arrayref. Identical semantics |
|
830
|
|
|
|
|
|
|
to the parent class's method of the same name (to which it delegates); |
|
831
|
|
|
|
|
|
|
see there for the croak conditions. |
|
832
|
|
|
|
|
|
|
|
|
833
|
|
|
|
|
|
|
=cut |
|
834
|
|
|
|
|
|
|
|
|
835
|
|
|
|
|
|
|
sub tagged_row_to_array { |
|
836
|
280
|
|
|
280
|
1
|
342
|
my $self = shift; |
|
837
|
280
|
|
|
|
|
512
|
return Algorithm::Classifier::IsolationForest::tagged_row_to_array( $self, @_ ); |
|
838
|
|
|
|
|
|
|
} |
|
839
|
|
|
|
|
|
|
|
|
840
|
|
|
|
|
|
|
=head2 munge_rows(\@rows) |
|
841
|
|
|
|
|
|
|
|
|
842
|
|
|
|
|
|
|
Applies the model's scalar mungers to positional rows, exactly as the |
|
843
|
|
|
|
|
|
|
parent class's method of the same name (to which it delegates); a model |
|
844
|
|
|
|
|
|
|
without C returns the input unchanged. |
|
845
|
|
|
|
|
|
|
|
|
846
|
|
|
|
|
|
|
=cut |
|
847
|
|
|
|
|
|
|
|
|
848
|
|
|
|
|
|
|
sub munge_rows { |
|
849
|
2
|
|
|
2
|
1
|
5
|
my $self = shift; |
|
850
|
2
|
|
|
|
|
10
|
return Algorithm::Classifier::IsolationForest::munge_rows( $self, @_ ); |
|
851
|
|
|
|
|
|
|
} |
|
852
|
|
|
|
|
|
|
|
|
853
|
|
|
|
|
|
|
=head1 MODEL SAVE/LOAD METHODS |
|
854
|
|
|
|
|
|
|
|
|
855
|
|
|
|
|
|
|
Persistence keeps the sliding window alongside the trees, so a reloaded |
|
856
|
|
|
|
|
|
|
model continues forgetting correctly as the stream resumes. This makes |
|
857
|
|
|
|
|
|
|
saved online models larger than batch models by O(window_size * |
|
858
|
|
|
|
|
|
|
n_features). Perl's RNG state is not persisted: a save/reload point |
|
859
|
|
|
|
|
|
|
breaks bit-for-bit reproducibility of subsequent learning versus an |
|
860
|
|
|
|
|
|
|
uninterrupted run, though scoring of the reloaded model is exact. |
|
861
|
|
|
|
|
|
|
|
|
862
|
|
|
|
|
|
|
=head2 to_json |
|
863
|
|
|
|
|
|
|
|
|
864
|
|
|
|
|
|
|
Returns a JSON representation of the model. |
|
865
|
|
|
|
|
|
|
|
|
866
|
|
|
|
|
|
|
my $json = $oif->to_json; |
|
867
|
|
|
|
|
|
|
|
|
868
|
|
|
|
|
|
|
=cut |
|
869
|
|
|
|
|
|
|
|
|
870
|
|
|
|
|
|
|
sub to_json { |
|
871
|
31
|
|
|
31
|
1
|
342
|
my ($self) = @_; |
|
872
|
|
|
|
|
|
|
my $payload = { |
|
873
|
|
|
|
|
|
|
format => 'Algorithm::Classifier::IsolationForest::Online', |
|
874
|
|
|
|
|
|
|
version => 1, |
|
875
|
|
|
|
|
|
|
params => { |
|
876
|
|
|
|
|
|
|
n_trees => $self->{n_trees}, |
|
877
|
|
|
|
|
|
|
window_size => $self->{window_size}, |
|
878
|
|
|
|
|
|
|
max_leaf_samples => $self->{max_leaf_samples}, |
|
879
|
|
|
|
|
|
|
growth => $self->{growth}, |
|
880
|
|
|
|
|
|
|
subsample => $self->{subsample}, |
|
881
|
|
|
|
|
|
|
contamination => $self->{contamination}, |
|
882
|
|
|
|
|
|
|
threshold => $self->{threshold}, |
|
883
|
|
|
|
|
|
|
n_features => $self->{n_features}, |
|
884
|
|
|
|
|
|
|
missing => $self->{missing}, |
|
885
|
|
|
|
|
|
|
feature_names => $self->{feature_names}, |
|
886
|
|
|
|
|
|
|
seen => $self->{seen}, |
|
887
|
|
|
|
|
|
|
mungers => $self->{mungers}, |
|
888
|
|
|
|
|
|
|
munger_module_version => $self->{munger_module_version}, |
|
889
|
|
|
|
|
|
|
schema_version => $self->{schema_version}, |
|
890
|
|
|
|
|
|
|
schema_description => $self->{schema_description}, |
|
891
|
|
|
|
|
|
|
feature_descriptions => $self->{feature_descriptions}, |
|
892
|
|
|
|
|
|
|
}, |
|
893
|
610
|
|
|
|
|
1696
|
trees => [ map { { count => $_->{count}, root => $_->{root} } } @{ $self->{trees} } ], |
|
|
31
|
|
|
|
|
200
|
|
|
894
|
|
|
|
|
|
|
window => $self->{window}, |
|
895
|
31
|
|
|
|
|
591
|
}; |
|
896
|
31
|
|
|
|
|
491
|
return JSON::PP->new->canonical(1)->encode($payload); |
|
897
|
|
|
|
|
|
|
} ## end sub to_json |
|
898
|
|
|
|
|
|
|
|
|
899
|
|
|
|
|
|
|
=head2 from_json($json) |
|
900
|
|
|
|
|
|
|
|
|
901
|
|
|
|
|
|
|
Init the object from the model in the specified JSON string. |
|
902
|
|
|
|
|
|
|
|
|
903
|
|
|
|
|
|
|
my $oif = Algorithm::Classifier::IsolationForest::Online->from_json($json); |
|
904
|
|
|
|
|
|
|
|
|
905
|
|
|
|
|
|
|
=cut |
|
906
|
|
|
|
|
|
|
|
|
907
|
|
|
|
|
|
|
sub from_json { |
|
908
|
18
|
|
|
18
|
1
|
9537
|
my ( $class, $text ) = @_; |
|
909
|
18
|
|
|
|
|
144
|
my $payload = JSON::PP->new->decode($text); |
|
910
|
|
|
|
|
|
|
croak "not an online IsolationForest model" |
|
911
|
|
|
|
|
|
|
unless ref $payload eq 'HASH' |
|
912
|
|
|
|
|
|
|
&& defined $payload->{format} |
|
913
|
18
|
100
|
33
|
|
|
1918571
|
&& $payload->{format} eq 'Algorithm::Classifier::IsolationForest::Online'; |
|
|
|
|
66
|
|
|
|
|
|
914
|
|
|
|
|
|
|
|
|
915
|
16
|
|
50
|
|
|
132
|
my $p = $payload->{params} || {}; |
|
916
|
|
|
|
|
|
|
|
|
917
|
|
|
|
|
|
|
my $self = { |
|
918
|
|
|
|
|
|
|
n_trees => $p->{n_trees}, |
|
919
|
|
|
|
|
|
|
window_size => $p->{window_size} // 0, |
|
920
|
|
|
|
|
|
|
max_leaf_samples => $p->{max_leaf_samples}, |
|
921
|
|
|
|
|
|
|
growth => $p->{growth} // 'adaptive', |
|
922
|
|
|
|
|
|
|
subsample => $p->{subsample} // 1.0, |
|
923
|
|
|
|
|
|
|
seed => undef, |
|
924
|
|
|
|
|
|
|
contamination => $p->{contamination}, |
|
925
|
|
|
|
|
|
|
threshold => $p->{threshold}, |
|
926
|
|
|
|
|
|
|
n_features => $p->{n_features}, |
|
927
|
|
|
|
|
|
|
missing => $p->{missing} // 'die', |
|
928
|
|
|
|
|
|
|
feature_names => $p->{feature_names}, |
|
929
|
|
|
|
|
|
|
# Recompiled lazily on first tagged use, like the parent. |
|
930
|
|
|
|
|
|
|
mungers => $p->{mungers}, |
|
931
|
|
|
|
|
|
|
munger_module_version => $p->{munger_module_version}, |
|
932
|
|
|
|
|
|
|
# Opaque schema metadata; absent in models saved before prototype |
|
933
|
|
|
|
|
|
|
# support, which just means "none recorded". |
|
934
|
|
|
|
|
|
|
schema_version => $p->{schema_version}, |
|
935
|
|
|
|
|
|
|
schema_description => $p->{schema_description}, |
|
936
|
|
|
|
|
|
|
feature_descriptions => $p->{feature_descriptions}, |
|
937
|
|
|
|
|
|
|
seen => $p->{seen} // 0, |
|
938
|
16
|
|
50
|
|
|
646
|
window => $payload->{window} // [], |
|
|
|
|
50
|
|
|
|
|
|
|
|
|
50
|
|
|
|
|
|
|
|
|
50
|
|
|
|
|
|
|
|
|
50
|
|
|
|
|
|
|
|
|
50
|
|
|
|
|
|
939
|
|
|
|
|
|
|
trees => [], |
|
940
|
|
|
|
|
|
|
_use_c => $Algorithm::Classifier::IsolationForest::HAS_C, |
|
941
|
|
|
|
|
|
|
_use_openmp => $Algorithm::Classifier::IsolationForest::HAS_OPENMP, |
|
942
|
|
|
|
|
|
|
}; |
|
943
|
|
|
|
|
|
|
|
|
944
|
16
|
|
|
|
|
45
|
my $trees = $payload->{trees}; |
|
945
|
16
|
50
|
33
|
|
|
147
|
croak "model contains no trees" unless ref $trees eq 'ARRAY' && @$trees; |
|
946
|
|
|
|
|
|
|
|
|
947
|
16
|
|
|
|
|
79
|
my $model = bless $self, $class; |
|
948
|
|
|
|
|
|
|
|
|
949
|
|
|
|
|
|
|
# depth_limit is a pure function of the tree's count, so recompute it |
|
950
|
|
|
|
|
|
|
# rather than trusting a stored float. |
|
951
|
|
|
|
|
|
|
$self->{trees} |
|
952
|
16
|
|
|
|
|
62
|
= [ map { { count => $_->{count}, root => $_->{root}, depth_limit => $model->_rpl( $_->{count} ) } } @$trees ]; |
|
|
400
|
|
|
|
|
804
|
|
|
953
|
|
|
|
|
|
|
|
|
954
|
16
|
|
|
|
|
1669
|
return $model; |
|
955
|
|
|
|
|
|
|
} ## end sub from_json |
|
956
|
|
|
|
|
|
|
|
|
957
|
|
|
|
|
|
|
=head2 save($path) |
|
958
|
|
|
|
|
|
|
|
|
959
|
|
|
|
|
|
|
Saves the model to the specified path. |
|
960
|
|
|
|
|
|
|
|
|
961
|
|
|
|
|
|
|
$oif->save($path); |
|
962
|
|
|
|
|
|
|
|
|
963
|
|
|
|
|
|
|
=cut |
|
964
|
|
|
|
|
|
|
|
|
965
|
|
|
|
|
|
|
sub save { |
|
966
|
11
|
|
|
11
|
1
|
936
|
my ( $self, $path ) = @_; |
|
967
|
11
|
|
|
|
|
74
|
write_file( $path, { 'atomic' => 1 }, $self->to_json ); |
|
968
|
|
|
|
|
|
|
} |
|
969
|
|
|
|
|
|
|
|
|
970
|
|
|
|
|
|
|
=head2 load($path) |
|
971
|
|
|
|
|
|
|
|
|
972
|
|
|
|
|
|
|
Init the object from the model in the specified file. |
|
973
|
|
|
|
|
|
|
|
|
974
|
|
|
|
|
|
|
my $oif = Algorithm::Classifier::IsolationForest::Online->load($path); |
|
975
|
|
|
|
|
|
|
|
|
976
|
|
|
|
|
|
|
=cut |
|
977
|
|
|
|
|
|
|
|
|
978
|
|
|
|
|
|
|
sub load { |
|
979
|
2
|
|
|
2
|
1
|
108197
|
my ( $class, $path ) = @_; |
|
980
|
2
|
|
|
|
|
11
|
my $raw_model = read_file($path); |
|
981
|
2
|
|
|
|
|
253
|
return $class->from_json($raw_model); |
|
982
|
|
|
|
|
|
|
} |
|
983
|
|
|
|
|
|
|
|
|
984
|
|
|
|
|
|
|
=head2 to_prototype |
|
985
|
|
|
|
|
|
|
|
|
986
|
|
|
|
|
|
|
Returns a prototype JSON string extracted from this model: its variable |
|
987
|
|
|
|
|
|
|
schema (feature_names, feature_descriptions, mungers, missing policy) |
|
988
|
|
|
|
|
|
|
plus its current tuning knobs, with C<"class": "online">. Identical |
|
989
|
|
|
|
|
|
|
semantics to the parent class's method -- see PROTOTYPES in |
|
990
|
|
|
|
|
|
|
L for the file format and the |
|
991
|
|
|
|
|
|
|
croak/placeholder rules. C is not emitted; pass it as an override |
|
992
|
|
|
|
|
|
|
when creating from the prototype. |
|
993
|
|
|
|
|
|
|
|
|
994
|
|
|
|
|
|
|
my $proto_json = $oif->to_prototype; |
|
995
|
|
|
|
|
|
|
|
|
996
|
|
|
|
|
|
|
=cut |
|
997
|
|
|
|
|
|
|
|
|
998
|
|
|
|
|
|
|
sub to_prototype { |
|
999
|
1
|
|
|
1
|
1
|
9
|
my ($self) = @_; |
|
1000
|
|
|
|
|
|
|
croak "to_prototype: this model has no feature_names; a prototype's variable " . "schema needs named features" |
|
1001
|
1
|
50
|
33
|
|
|
7
|
unless ref $self->{feature_names} eq 'ARRAY' && @{ $self->{feature_names} }; |
|
|
1
|
|
|
|
|
17
|
|
|
1002
|
|
|
|
|
|
|
|
|
1003
|
|
|
|
|
|
|
my $schema = { |
|
1004
|
|
|
|
|
|
|
feature_names => $self->{feature_names}, |
|
1005
|
|
|
|
|
|
|
missing => $self->{missing}, |
|
1006
|
1
|
|
|
|
|
4
|
}; |
|
1007
|
|
|
|
|
|
|
$schema->{feature_descriptions} = $self->{feature_descriptions} |
|
1008
|
1
|
50
|
33
|
|
|
4
|
if ref $self->{feature_descriptions} eq 'HASH' && %{ $self->{feature_descriptions} }; |
|
|
1
|
|
|
|
|
4
|
|
|
1009
|
|
|
|
|
|
|
$schema->{mungers} = $self->{mungers} |
|
1010
|
1
|
50
|
33
|
|
|
4
|
if ref $self->{mungers} eq 'HASH' && %{ $self->{mungers} }; |
|
|
0
|
|
|
|
|
0
|
|
|
1011
|
|
|
|
|
|
|
|
|
1012
|
|
|
|
|
|
|
my $params = { |
|
1013
|
|
|
|
|
|
|
n_trees => $self->{n_trees}, |
|
1014
|
|
|
|
|
|
|
window_size => $self->{window_size}, |
|
1015
|
|
|
|
|
|
|
max_leaf_samples => $self->{max_leaf_samples}, |
|
1016
|
|
|
|
|
|
|
growth => $self->{growth}, |
|
1017
|
|
|
|
|
|
|
subsample => $self->{subsample}, |
|
1018
|
1
|
|
|
|
|
4
|
}; |
|
1019
|
1
|
50
|
|
|
|
3
|
$params->{contamination} = $self->{contamination} if defined $self->{contamination}; |
|
1020
|
|
|
|
|
|
|
|
|
1021
|
|
|
|
|
|
|
return JSON::PP->new->canonical(1)->encode( |
|
1022
|
|
|
|
|
|
|
{ |
|
1023
|
|
|
|
|
|
|
format => 'Algorithm::Classifier::IsolationForest::Prototype', |
|
1024
|
|
|
|
|
|
|
version => 1, |
|
1025
|
|
|
|
|
|
|
class => 'online', |
|
1026
|
|
|
|
|
|
|
schema_version => $self->{schema_version} // '0', |
|
1027
|
|
|
|
|
|
|
schema_description => $self->{schema_description} |
|
1028
|
1
|
|
50
|
|
|
8
|
// '(none recorded; describe this schema and bump schema_version)', |
|
|
|
|
50
|
|
|
|
|
|
1029
|
|
|
|
|
|
|
schema => $schema, |
|
1030
|
|
|
|
|
|
|
params => $params, |
|
1031
|
|
|
|
|
|
|
} |
|
1032
|
|
|
|
|
|
|
); |
|
1033
|
|
|
|
|
|
|
} ## end sub to_prototype |
|
1034
|
|
|
|
|
|
|
|
|
1035
|
|
|
|
|
|
|
=head1 REFERENCES |
|
1036
|
|
|
|
|
|
|
|
|
1037
|
|
|
|
|
|
|
Filippo Leveni, Guilherme Weigert Cassales, Bernhard Pfahringer, Albert |
|
1038
|
|
|
|
|
|
|
Bifet, Giacomo Boracchi (2024). Online Isolation Forest. |
|
1039
|
|
|
|
|
|
|
|
|
1040
|
|
|
|
|
|
|
L |
|
1041
|
|
|
|
|
|
|
|
|
1042
|
|
|
|
|
|
|
L |
|
1043
|
|
|
|
|
|
|
|
|
1044
|
|
|
|
|
|
|
L |
|
1045
|
|
|
|
|
|
|
|
|
1046
|
|
|
|
|
|
|
=cut |
|
1047
|
|
|
|
|
|
|
|
|
1048
|
|
|
|
|
|
|
### |
|
1049
|
|
|
|
|
|
|
### |
|
1050
|
|
|
|
|
|
|
### internal stuff below |
|
1051
|
|
|
|
|
|
|
### |
|
1052
|
|
|
|
|
|
|
### |
|
1053
|
|
|
|
|
|
|
|
|
1054
|
|
|
|
|
|
|
sub _check_learned { |
|
1055
|
97
|
|
|
97
|
|
189
|
my ($self) = @_; |
|
1056
|
|
|
|
|
|
|
croak "model has not learned any data yet; call learn() first" |
|
1057
|
97
|
100
|
|
|
|
602
|
unless $self->{seen} > 0; |
|
1058
|
|
|
|
|
|
|
} |
|
1059
|
|
|
|
|
|
|
|
|
1060
|
|
|
|
|
|
|
# Validate one incoming sample, apply the missing-value strategy, and |
|
1061
|
|
|
|
|
|
|
# return a fresh dense copy (the window owns its rows; the caller may |
|
1062
|
|
|
|
|
|
|
# reuse or mutate the original). Locks in n_features on first contact. |
|
1063
|
|
|
|
|
|
|
sub _prep_row { |
|
1064
|
16418
|
|
|
16418
|
|
25665
|
my ( $self, $row, $caller ) = @_; |
|
1065
|
16418
|
50
|
33
|
|
|
44247
|
croak "$caller: each sample must be an arrayref of features" |
|
1066
|
|
|
|
|
|
|
unless ref $row eq 'ARRAY' && @$row; |
|
1067
|
|
|
|
|
|
|
|
|
1068
|
16418
|
100
|
|
|
|
34800
|
if ( !defined $self->{n_features} ) { |
|
|
|
100
|
|
|
|
|
|
|
1069
|
48
|
|
|
|
|
126
|
$self->{n_features} = scalar @$row; |
|
1070
|
|
|
|
|
|
|
} elsif ( scalar @$row != $self->{n_features} ) { |
|
1071
|
2
|
|
|
|
|
409
|
croak "$caller: sample has " . scalar(@$row) . " features but model expects " . $self->{n_features}; |
|
1072
|
|
|
|
|
|
|
} |
|
1073
|
|
|
|
|
|
|
|
|
1074
|
16416
|
100
|
|
|
|
27457
|
if ( $self->{missing} eq 'die' ) { |
|
1075
|
15366
|
|
|
|
|
24247
|
for my $f ( 0 .. $#$row ) { |
|
1076
|
31768
|
100
|
|
|
|
51315
|
next if defined $row->[$f]; |
|
1077
|
1
|
|
|
|
|
98
|
croak "$caller: undef feature value at column $f; " |
|
1078
|
|
|
|
|
|
|
. "construct with missing => 'zero' to learn from data with missing values"; |
|
1079
|
|
|
|
|
|
|
} |
|
1080
|
15365
|
|
|
|
|
31750
|
return [@$row]; |
|
1081
|
|
|
|
|
|
|
} |
|
1082
|
|
|
|
|
|
|
|
|
1083
|
|
|
|
|
|
|
# zero: a missing cell counts as the value 0. |
|
1084
|
1050
|
|
100
|
|
|
2145
|
return [ map { $_ // 0 } @$row ]; |
|
|
2100
|
|
|
|
|
6485
|
|
|
1085
|
|
|
|
|
|
|
} ## end sub _prep_row |
|
1086
|
|
|
|
|
|
|
|
|
1087
|
|
|
|
|
|
|
# The depth budget for n points: how deep a tree fed n points is allowed |
|
1088
|
|
|
|
|
|
|
# (learn) or expected (scoring normalisation, per-leaf adjustment) to |
|
1089
|
|
|
|
|
|
|
# go. log base 4 = log(2 * branching_factor) with binary trees. Under |
|
1090
|
|
|
|
|
|
|
# max_leaf_samples points there is nothing to isolate: 0. |
|
1091
|
|
|
|
|
|
|
sub _rpl { |
|
1092
|
111293
|
|
|
111293
|
|
143150
|
my ( $self, $n ) = @_; |
|
1093
|
111293
|
|
|
|
|
137724
|
my $eta = $self->{max_leaf_samples}; |
|
1094
|
111293
|
100
|
|
|
|
157014
|
return 0 if $n < $eta; |
|
1095
|
105613
|
|
|
|
|
175911
|
return log( $n / $eta ) / _LOG4; |
|
1096
|
|
|
|
|
|
|
} |
|
1097
|
|
|
|
|
|
|
|
|
1098
|
|
|
|
|
|
|
# How many points a node at $depth needs before it may split (or below |
|
1099
|
|
|
|
|
|
|
# which, on forgetting, it collapses back into a leaf). |
|
1100
|
|
|
|
|
|
|
sub _split_threshold { |
|
1101
|
86917
|
|
|
86917
|
|
114639
|
my ( $self, $depth ) = @_; |
|
1102
|
86917
|
100
|
|
|
|
206928
|
return $self->{max_leaf_samples} * ( $self->{growth} eq 'adaptive' ? 2**$depth : 1 ); |
|
1103
|
|
|
|
|
|
|
} |
|
1104
|
|
|
|
|
|
|
|
|
1105
|
|
|
|
|
|
|
# Number of points the model currently reflects: the window fill, or the |
|
1106
|
|
|
|
|
|
|
# whole stream when forgetting is disabled. |
|
1107
|
|
|
|
|
|
|
sub _data_size { |
|
1108
|
2006
|
|
|
2006
|
|
2809
|
my ($self) = @_; |
|
1109
|
2006
|
100
|
|
|
|
3716
|
return $self->{window_size} ? scalar @{ $self->{window} } : $self->{seen}; |
|
|
1904
|
|
|
|
|
4253
|
|
|
1110
|
|
|
|
|
|
|
} |
|
1111
|
|
|
|
|
|
|
|
|
1112
|
|
|
|
|
|
|
# exp() multiplier turning a per-sample depth SUM into the normalised |
|
1113
|
|
|
|
|
|
|
# anomaly score: 2**(-(sum/t)/norm) == exp(-sum * log(2)/(t*norm)). |
|
1114
|
|
|
|
|
|
|
# _EPS keeps a zero normaliser (fewer than max_leaf_samples points seen) |
|
1115
|
|
|
|
|
|
|
# well-defined; every depth is 0 then, so everything scores 1.0. |
|
1116
|
|
|
|
|
|
|
sub _score_inv { |
|
1117
|
2006
|
|
|
2006
|
|
2958
|
my ($self) = @_; |
|
1118
|
2006
|
|
|
|
|
3139
|
my $norm = $self->_rpl( $self->_data_size * $self->{subsample} ); |
|
1119
|
2006
|
|
|
|
|
5411
|
return _LOG2 / ( $self->{n_trees} * ( $norm + _EPS ) ); |
|
1120
|
|
|
|
|
|
|
} |
|
1121
|
|
|
|
|
|
|
|
|
1122
|
|
|
|
|
|
|
#------------------------------------------------------------------------------- |
|
1123
|
|
|
|
|
|
|
# Learning. |
|
1124
|
|
|
|
|
|
|
#------------------------------------------------------------------------------- |
|
1125
|
|
|
|
|
|
|
|
|
1126
|
|
|
|
|
|
|
# Advance the stream by one (already prepped) row: every tree learns it |
|
1127
|
|
|
|
|
|
|
# (subject to subsampling), it enters the window, and the oldest point |
|
1128
|
|
|
|
|
|
|
# beyond the window is forgotten. This is the single choke point through |
|
1129
|
|
|
|
|
|
|
# which every tree mutation flows, so it is also where the packed C |
|
1130
|
|
|
|
|
|
|
# scoring snapshot gets invalidated. |
|
1131
|
|
|
|
|
|
|
# |
|
1132
|
|
|
|
|
|
|
# With use_c the per-tree learn and eviction loops run inside the |
|
1133
|
|
|
|
|
|
|
# parent's C backend (online_learn_row_xs / online_unlearn_row_xs), |
|
1134
|
|
|
|
|
|
|
# mutating the same live trees this file's Perl recursion would. Random |
|
1135
|
|
|
|
|
|
|
# draws go through the same generator in the same order, so the trees |
|
1136
|
|
|
|
|
|
|
# built are bit-identical either way (on nvsize == 8 perls) -- use_c |
|
1137
|
|
|
|
|
|
|
# only changes speed, matching fit()'s guarantee. |
|
1138
|
|
|
|
|
|
|
sub _learn_row { |
|
1139
|
16415
|
|
|
16415
|
|
23627
|
my ( $self, $r ) = @_; |
|
1140
|
16415
|
|
|
|
|
21752
|
my $sub = $self->{subsample}; |
|
1141
|
|
|
|
|
|
|
|
|
1142
|
16415
|
|
|
|
|
31262
|
$self->_invalidate_c_trees; |
|
1143
|
|
|
|
|
|
|
|
|
1144
|
16415
|
100
|
|
|
|
24050
|
if ( _HAS_ONLINE_XS && $self->{_use_c} ) { |
|
1145
|
|
|
|
|
|
|
Algorithm::Classifier::IsolationForest::online_learn_row_xs( |
|
1146
|
|
|
|
|
|
|
$self->{trees}, $r, $self->{n_features}, |
|
1147
|
|
|
|
|
|
|
$self->{max_leaf_samples}, |
|
1148
|
13415
|
100
|
|
|
|
337326
|
( $self->{growth} eq 'adaptive' ? 1 : 0 ), $sub |
|
1149
|
|
|
|
|
|
|
); |
|
1150
|
|
|
|
|
|
|
} else { |
|
1151
|
3000
|
|
|
|
|
3958
|
for my $tree ( @{ $self->{trees} } ) { |
|
|
3000
|
|
|
|
|
4980
|
|
|
1152
|
46500
|
100
|
100
|
|
|
79023
|
next if $sub < 1 && rand() >= $sub; |
|
1153
|
44471
|
|
|
|
|
62529
|
$self->_tree_learn( $tree, $r ); |
|
1154
|
|
|
|
|
|
|
} |
|
1155
|
|
|
|
|
|
|
} |
|
1156
|
16415
|
|
|
|
|
24387
|
$self->{seen}++; |
|
1157
|
|
|
|
|
|
|
|
|
1158
|
16415
|
100
|
|
|
|
27513
|
if ( $self->{window_size} ) { |
|
1159
|
15115
|
|
|
|
|
17682
|
push @{ $self->{window} }, $r; |
|
|
15115
|
|
|
|
|
23659
|
|
|
1160
|
15115
|
100
|
|
|
|
18812
|
if ( @{ $self->{window} } > $self->{window_size} ) { |
|
|
15115
|
|
|
|
|
26372
|
|
|
1161
|
8463
|
|
|
|
|
10213
|
my $old = shift @{ $self->{window} }; |
|
|
8463
|
|
|
|
|
12103
|
|
|
1162
|
8463
|
100
|
|
|
|
13858
|
if ( _HAS_ONLINE_XS && $self->{_use_c} ) { |
|
1163
|
|
|
|
|
|
|
Algorithm::Classifier::IsolationForest::online_unlearn_row_xs( |
|
1164
|
|
|
|
|
|
|
$self->{trees}, $old, $self->{n_features}, |
|
1165
|
|
|
|
|
|
|
$self->{max_leaf_samples}, |
|
1166
|
7003
|
100
|
|
|
|
407887
|
( $self->{growth} eq 'adaptive' ? 1 : 0 ), $sub |
|
1167
|
|
|
|
|
|
|
); |
|
1168
|
|
|
|
|
|
|
} else { |
|
1169
|
1460
|
|
|
|
|
1957
|
for my $tree ( @{ $self->{trees} } ) { |
|
|
1460
|
|
|
|
|
2386
|
|
|
1170
|
22760
|
100
|
100
|
|
|
40718
|
next if $sub < 1 && rand() >= $sub; |
|
1171
|
21270
|
|
|
|
|
42846
|
$self->_tree_unlearn( $tree, $old ); |
|
1172
|
|
|
|
|
|
|
} |
|
1173
|
|
|
|
|
|
|
} |
|
1174
|
|
|
|
|
|
|
} ## end if ( @{ $self->{window} } > $self->{window_size...}) |
|
1175
|
|
|
|
|
|
|
} ## end if ( $self->{window_size} ) |
|
1176
|
16415
|
|
|
|
|
29342
|
return; |
|
1177
|
|
|
|
|
|
|
} ## end sub _learn_row |
|
1178
|
|
|
|
|
|
|
|
|
1179
|
|
|
|
|
|
|
sub _tree_learn { |
|
1180
|
44471
|
|
|
44471
|
|
60105
|
my ( $self, $tree, $x ) = @_; |
|
1181
|
44471
|
|
|
|
|
55413
|
$tree->{count}++; |
|
1182
|
44471
|
|
|
|
|
66600
|
$tree->{depth_limit} = $self->_rpl( $tree->{count} ); |
|
1183
|
44471
|
100
|
|
|
|
65214
|
if ( !defined $tree->{root} ) { |
|
1184
|
110
|
|
|
|
|
223
|
$tree->{root} = [ _NT_LEAF, 1, [@$x], [@$x] ]; |
|
1185
|
|
|
|
|
|
|
} else { |
|
1186
|
44361
|
|
|
|
|
65636
|
$tree->{root} = $self->_node_learn( $tree->{root}, $x, 0, $tree->{depth_limit} ); |
|
1187
|
|
|
|
|
|
|
} |
|
1188
|
44471
|
|
|
|
|
61660
|
return; |
|
1189
|
|
|
|
|
|
|
} ## end sub _tree_learn |
|
1190
|
|
|
|
|
|
|
|
|
1191
|
|
|
|
|
|
|
# Route $x down to its leaf, growing counts and bounding boxes along the |
|
1192
|
|
|
|
|
|
|
# path. A leaf that has accumulated its split requirement (and still has |
|
1193
|
|
|
|
|
|
|
# depth budget) is replaced by a subtree built from synthetic points |
|
1194
|
|
|
|
|
|
|
# sampled inside its box -- the return value replaces the node in the |
|
1195
|
|
|
|
|
|
|
# parent, which is how leaves turn into subtrees in place. |
|
1196
|
|
|
|
|
|
|
sub _node_learn { |
|
1197
|
125393
|
|
|
125393
|
|
174724
|
my ( $self, $node, $x, $depth, $limit ) = @_; |
|
1198
|
|
|
|
|
|
|
|
|
1199
|
125393
|
|
|
|
|
144955
|
$node->[_N_COUNT]++; |
|
1200
|
125393
|
50
|
|
|
|
163888
|
if ( !defined $node->[_N_LO] ) { |
|
1201
|
|
|
|
|
|
|
|
|
1202
|
|
|
|
|
|
|
# Leaf born from an empty synthetic partition: first real point |
|
1203
|
|
|
|
|
|
|
# initialises the box. |
|
1204
|
0
|
|
|
|
|
0
|
$node->[_N_LO] = [@$x]; |
|
1205
|
0
|
|
|
|
|
0
|
$node->[_N_HI] = [@$x]; |
|
1206
|
|
|
|
|
|
|
} else { |
|
1207
|
125393
|
|
|
|
|
164541
|
my ( $lo, $hi ) = ( $node->[_N_LO], $node->[_N_HI] ); |
|
1208
|
125393
|
|
|
|
|
176330
|
for my $f ( 0 .. $#$x ) { |
|
1209
|
250786
|
|
|
|
|
287220
|
my $v = $x->[$f]; |
|
1210
|
250786
|
100
|
|
|
|
353755
|
$lo->[$f] = $v if $v < $lo->[$f]; |
|
1211
|
250786
|
100
|
|
|
|
393731
|
$hi->[$f] = $v if $v > $hi->[$f]; |
|
1212
|
|
|
|
|
|
|
} |
|
1213
|
|
|
|
|
|
|
} |
|
1214
|
|
|
|
|
|
|
|
|
1215
|
125393
|
100
|
|
|
|
185054
|
if ( $node->[_N_TYPE] == _NT_LEAF ) { |
|
1216
|
44361
|
100
|
100
|
|
|
65667
|
if ( $node->[_N_COUNT] >= $self->_split_threshold($depth) && $depth < $limit ) { |
|
1217
|
449
|
|
|
|
|
987
|
my $pts = $self->_sample_box( $node->[_N_LO], $node->[_N_HI], $node->[_N_COUNT] ); |
|
1218
|
449
|
|
|
|
|
964
|
return $self->_build_from_points( $pts, $depth, $limit ); |
|
1219
|
|
|
|
|
|
|
} |
|
1220
|
43912
|
|
|
|
|
65179
|
return $node; |
|
1221
|
|
|
|
|
|
|
} |
|
1222
|
|
|
|
|
|
|
|
|
1223
|
81032
|
100
|
|
|
|
121162
|
my $ci = $x->[ $node->[_N_ATTR] ] < $node->[_N_SPLIT] ? _N_LEFT : _N_RIGHT; |
|
1224
|
81032
|
|
|
|
|
121198
|
$node->[$ci] = $self->_node_learn( $node->[$ci], $x, $depth + 1, $limit ); |
|
1225
|
81032
|
|
|
|
|
114056
|
return $node; |
|
1226
|
|
|
|
|
|
|
} ## end sub _node_learn |
|
1227
|
|
|
|
|
|
|
|
|
1228
|
|
|
|
|
|
|
# $n synthetic points drawn uniformly inside the box -- the stand-in for |
|
1229
|
|
|
|
|
|
|
# the real points the tree never stored. |
|
1230
|
|
|
|
|
|
|
sub _sample_box { |
|
1231
|
449
|
|
|
449
|
|
782
|
my ( $self, $lo, $hi, $n ) = @_; |
|
1232
|
449
|
|
|
|
|
652
|
my @pts; |
|
1233
|
449
|
|
|
|
|
798
|
for ( 1 .. $n ) { |
|
1234
|
15686
|
50
|
|
|
|
20873
|
push @pts, [ map { my $w = $hi->[$_] - $lo->[$_]; $w > 0 ? $lo->[$_] + rand() * $w : $lo->[$_] } 0 .. $#$lo ]; |
|
|
31372
|
|
|
|
|
37620
|
|
|
|
31372
|
|
|
|
|
56529
|
|
|
1235
|
|
|
|
|
|
|
} |
|
1236
|
449
|
|
|
|
|
809
|
return \@pts; |
|
1237
|
|
|
|
|
|
|
} |
|
1238
|
|
|
|
|
|
|
|
|
1239
|
|
|
|
|
|
|
# Recursively build a subtree over (synthetic) points: random feature, |
|
1240
|
|
|
|
|
|
|
# uniform split value within the points' range on it, recurse on the |
|
1241
|
|
|
|
|
|
|
# partitions. Leaves keep the partition's count and box. |
|
1242
|
|
|
|
|
|
|
sub _build_from_points { |
|
1243
|
1347
|
|
|
1347
|
|
2304
|
my ( $self, $pts, $depth, $limit ) = @_; |
|
1244
|
1347
|
|
|
|
|
1726
|
my $n = scalar @$pts; |
|
1245
|
1347
|
|
|
|
|
2064
|
my ( $lo, $hi ) = _box_of($pts); |
|
1246
|
|
|
|
|
|
|
|
|
1247
|
1347
|
100
|
100
|
|
|
2465
|
if ( $n < $self->_split_threshold($depth) || $depth >= $limit ) { |
|
1248
|
898
|
|
|
|
|
2097
|
return [ _NT_LEAF, $n, $lo, $hi ]; |
|
1249
|
|
|
|
|
|
|
} |
|
1250
|
|
|
|
|
|
|
|
|
1251
|
449
|
|
|
|
|
980
|
my $attr = int( rand( $self->{n_features} ) ); |
|
1252
|
449
|
|
|
|
|
842
|
my ( $pmin, $pmax ) = ( $pts->[0][$attr], $pts->[0][$attr] ); |
|
1253
|
449
|
|
|
|
|
720
|
for my $p (@$pts) { |
|
1254
|
15686
|
100
|
|
|
|
21469
|
$pmin = $p->[$attr] if $p->[$attr] < $pmin; |
|
1255
|
15686
|
100
|
|
|
|
22693
|
$pmax = $p->[$attr] if $p->[$attr] > $pmax; |
|
1256
|
|
|
|
|
|
|
} |
|
1257
|
449
|
|
|
|
|
757
|
my $split = $pmin + rand() * ( $pmax - $pmin ); |
|
1258
|
|
|
|
|
|
|
|
|
1259
|
449
|
|
|
|
|
652
|
my ( @l, @r ); |
|
1260
|
449
|
|
|
|
|
642
|
for my $p (@$pts) { |
|
1261
|
15686
|
100
|
|
|
|
19891
|
if ( $p->[$attr] < $split ) { push @l, $p } |
|
|
7769
|
|
|
|
|
9579
|
|
|
1262
|
7917
|
|
|
|
|
10484
|
else { push @r, $p } |
|
1263
|
|
|
|
|
|
|
} |
|
1264
|
|
|
|
|
|
|
|
|
1265
|
449
|
|
|
|
|
1001
|
my $left = $self->_build_from_points( \@l, $depth + 1, $limit ); |
|
1266
|
449
|
|
|
|
|
938
|
my $right = $self->_build_from_points( \@r, $depth + 1, $limit ); |
|
1267
|
449
|
|
|
|
|
3712
|
return [ _NT_AXIS, $n, $lo, $hi, $attr, $split, $left, $right ]; |
|
1268
|
|
|
|
|
|
|
} ## end sub _build_from_points |
|
1269
|
|
|
|
|
|
|
|
|
1270
|
|
|
|
|
|
|
#------------------------------------------------------------------------------- |
|
1271
|
|
|
|
|
|
|
# Forgetting. |
|
1272
|
|
|
|
|
|
|
#------------------------------------------------------------------------------- |
|
1273
|
|
|
|
|
|
|
|
|
1274
|
|
|
|
|
|
|
sub _tree_unlearn { |
|
1275
|
21270
|
|
|
21270
|
|
30699
|
my ( $self, $tree, $x ) = @_; |
|
1276
|
21270
|
|
|
|
|
27842
|
$tree->{count}--; |
|
1277
|
21270
|
|
|
|
|
32638
|
$tree->{depth_limit} = $self->_rpl( $tree->{count} ); |
|
1278
|
21270
|
50
|
|
|
|
34824
|
return unless defined $tree->{root}; |
|
1279
|
21270
|
|
|
|
|
32670
|
$tree->{root} = $self->_node_unlearn( $tree->{root}, $x, 0 ); |
|
1280
|
21270
|
|
|
|
|
31705
|
return; |
|
1281
|
|
|
|
|
|
|
} |
|
1282
|
|
|
|
|
|
|
|
|
1283
|
|
|
|
|
|
|
# Route the forgotten point down its (current) path, decrementing counts. |
|
1284
|
|
|
|
|
|
|
# An internal node whose count no longer justifies its split collapses |
|
1285
|
|
|
|
|
|
|
# back into a leaf; otherwise its box is refreshed to the union of its |
|
1286
|
|
|
|
|
|
|
# children's, which is how boxes shrink as old extremes age out. |
|
1287
|
|
|
|
|
|
|
sub _node_unlearn { |
|
1288
|
62358
|
|
|
62358
|
|
88615
|
my ( $self, $node, $x, $depth ) = @_; |
|
1289
|
|
|
|
|
|
|
|
|
1290
|
62358
|
|
|
|
|
75652
|
$node->[_N_COUNT]--; |
|
1291
|
62358
|
100
|
|
|
|
100181
|
return $node if $node->[_N_TYPE] == _NT_LEAF; |
|
1292
|
41209
|
100
|
|
|
|
60135
|
return _collapse($node) if $node->[_N_COUNT] < $self->_split_threshold($depth); |
|
1293
|
|
|
|
|
|
|
|
|
1294
|
41088
|
100
|
|
|
|
66558
|
my $ci = $x->[ $node->[_N_ATTR] ] < $node->[_N_SPLIT] ? _N_LEFT : _N_RIGHT; |
|
1295
|
41088
|
|
|
|
|
62139
|
$node->[$ci] = $self->_node_unlearn( $node->[$ci], $x, $depth + 1 ); |
|
1296
|
|
|
|
|
|
|
|
|
1297
|
41088
|
|
|
|
|
61348
|
my ( $lo, $hi ) = _box_union( $node->[_N_LEFT], $node->[_N_RIGHT] ); |
|
1298
|
41088
|
50
|
|
|
|
62377
|
if ( defined $lo ) { |
|
1299
|
41088
|
|
|
|
|
57411
|
$node->[_N_LO] = $lo; |
|
1300
|
41088
|
|
|
|
|
54564
|
$node->[_N_HI] = $hi; |
|
1301
|
|
|
|
|
|
|
} |
|
1302
|
41088
|
|
|
|
|
60272
|
return $node; |
|
1303
|
|
|
|
|
|
|
} ## end sub _node_unlearn |
|
1304
|
|
|
|
|
|
|
|
|
1305
|
|
|
|
|
|
|
# Aggregate a subtree back into a single leaf holding the subtree's |
|
1306
|
|
|
|
|
|
|
# (already decremented) count and the union of its descendants' boxes. |
|
1307
|
|
|
|
|
|
|
sub _collapse { |
|
1308
|
363
|
|
|
363
|
|
484
|
my ($node) = @_; |
|
1309
|
363
|
100
|
|
|
|
727
|
return $node if $node->[_N_TYPE] == _NT_LEAF; |
|
1310
|
121
|
|
|
|
|
226
|
my $l = _collapse( $node->[_N_LEFT] ); |
|
1311
|
121
|
|
|
|
|
248
|
my $r = _collapse( $node->[_N_RIGHT] ); |
|
1312
|
121
|
|
|
|
|
245
|
my ( $lo, $hi ) = _box_union( $l, $r ); |
|
1313
|
121
|
50
|
|
|
|
235
|
if ( !defined $lo ) { |
|
1314
|
|
|
|
|
|
|
|
|
1315
|
|
|
|
|
|
|
# Both children empty: keep the node's own box. |
|
1316
|
0
|
|
|
|
|
0
|
( $lo, $hi ) = ( $node->[_N_LO], $node->[_N_HI] ); |
|
1317
|
|
|
|
|
|
|
} |
|
1318
|
121
|
|
|
|
|
532
|
return [ _NT_LEAF, $node->[_N_COUNT], $lo, $hi ]; |
|
1319
|
|
|
|
|
|
|
} ## end sub _collapse |
|
1320
|
|
|
|
|
|
|
|
|
1321
|
|
|
|
|
|
|
# (lo, hi) of the union of two nodes' boxes, as fresh arrays (parent |
|
1322
|
|
|
|
|
|
|
# boxes grow in place, so they must never alias a child's). Nodes with |
|
1323
|
|
|
|
|
|
|
# no box yet (empty leaves) are skipped; (undef, undef) if neither has |
|
1324
|
|
|
|
|
|
|
# one. |
|
1325
|
|
|
|
|
|
|
sub _box_union { |
|
1326
|
41209
|
|
|
41209
|
|
54079
|
my ( $a, $b ) = @_; |
|
1327
|
41209
|
|
|
|
|
55304
|
my @boxed = grep { defined $_->[_N_LO] } ( $a, $b ); |
|
|
82418
|
|
|
|
|
124089
|
|
|
1328
|
41209
|
50
|
|
|
|
58568
|
return ( undef, undef ) unless @boxed; |
|
1329
|
41209
|
|
|
|
|
46233
|
my $lo = [ @{ $boxed[0][_N_LO] } ]; |
|
|
41209
|
|
|
|
|
61462
|
|
|
1330
|
41209
|
|
|
|
|
48849
|
my $hi = [ @{ $boxed[0][_N_HI] } ]; |
|
|
41209
|
|
|
|
|
60213
|
|
|
1331
|
41209
|
50
|
|
|
|
66043
|
if ( @boxed == 2 ) { |
|
1332
|
41209
|
|
|
|
|
57574
|
my ( $blo, $bhi ) = ( $boxed[1][_N_LO], $boxed[1][_N_HI] ); |
|
1333
|
41209
|
|
|
|
|
61417
|
for my $f ( 0 .. $#$lo ) { |
|
1334
|
82418
|
100
|
|
|
|
126956
|
$lo->[$f] = $blo->[$f] if $blo->[$f] < $lo->[$f]; |
|
1335
|
82418
|
100
|
|
|
|
140833
|
$hi->[$f] = $bhi->[$f] if $bhi->[$f] > $hi->[$f]; |
|
1336
|
|
|
|
|
|
|
} |
|
1337
|
|
|
|
|
|
|
} |
|
1338
|
41209
|
|
|
|
|
69647
|
return ( $lo, $hi ); |
|
1339
|
|
|
|
|
|
|
} ## end sub _box_union |
|
1340
|
|
|
|
|
|
|
|
|
1341
|
|
|
|
|
|
|
# (lo, hi) bounding box of a point set; (undef, undef) when empty. |
|
1342
|
|
|
|
|
|
|
sub _box_of { |
|
1343
|
1347
|
|
|
1347
|
|
1858
|
my ($pts) = @_; |
|
1344
|
1347
|
50
|
|
|
|
2327
|
return ( undef, undef ) unless @$pts; |
|
1345
|
1347
|
|
|
|
|
1589
|
my $lo = [ @{ $pts->[0] } ]; |
|
|
1347
|
|
|
|
|
2395
|
|
|
1346
|
1347
|
|
|
|
|
1715
|
my $hi = [ @{ $pts->[0] } ]; |
|
|
1347
|
|
|
|
|
2195
|
|
|
1347
|
1347
|
|
|
|
|
1948
|
for my $p (@$pts) { |
|
1348
|
31372
|
|
|
|
|
42056
|
for my $f ( 0 .. $#$p ) { |
|
1349
|
62744
|
100
|
|
|
|
89080
|
$lo->[$f] = $p->[$f] if $p->[$f] < $lo->[$f]; |
|
1350
|
62744
|
100
|
|
|
|
101196
|
$hi->[$f] = $p->[$f] if $p->[$f] > $hi->[$f]; |
|
1351
|
|
|
|
|
|
|
} |
|
1352
|
|
|
|
|
|
|
} |
|
1353
|
1347
|
|
|
|
|
2620
|
return ( $lo, $hi ); |
|
1354
|
|
|
|
|
|
|
} ## end sub _box_of |
|
1355
|
|
|
|
|
|
|
|
|
1356
|
|
|
|
|
|
|
#------------------------------------------------------------------------------- |
|
1357
|
|
|
|
|
|
|
# Scoring. |
|
1358
|
|
|
|
|
|
|
#------------------------------------------------------------------------------- |
|
1359
|
|
|
|
|
|
|
|
|
1360
|
|
|
|
|
|
|
# Depth of the leaf $x lands in, plus the leaf's own depth budget -- the |
|
1361
|
|
|
|
|
|
|
# streaming analogue of the batch scorer's c(leaf size) adjustment. |
|
1362
|
|
|
|
|
|
|
# Scoring tolerates undef cells (mapped to 0), matching the parent class. |
|
1363
|
|
|
|
|
|
|
sub _depth_of { |
|
1364
|
39080
|
|
|
39080
|
|
51022
|
my ( $self, $x, $node ) = @_; |
|
1365
|
39080
|
|
|
|
|
42268
|
my $depth = 0; |
|
1366
|
39080
|
|
|
|
|
52414
|
while ( $node->[_N_TYPE] ) { |
|
1367
|
98226
|
100
|
100
|
|
|
154877
|
$node = ( $x->[ $node->[_N_ATTR] ] // 0 ) < $node->[_N_SPLIT] ? $node->[_N_LEFT] : $node->[_N_RIGHT]; |
|
1368
|
98226
|
|
|
|
|
131040
|
$depth++; |
|
1369
|
|
|
|
|
|
|
} |
|
1370
|
39080
|
|
|
|
|
51329
|
return $depth + $self->_rpl( $node->[_N_COUNT] ); |
|
1371
|
|
|
|
|
|
|
} |
|
1372
|
|
|
|
|
|
|
|
|
1373
|
|
|
|
|
|
|
# Per-sample depth sums across all trees (tree-outer, sample-inner for |
|
1374
|
|
|
|
|
|
|
# cache locality, mirroring the parent's pure-Perl loops). |
|
1375
|
|
|
|
|
|
|
sub _depth_sums { |
|
1376
|
14
|
|
|
14
|
|
40
|
my ( $self, $data ) = @_; |
|
1377
|
14
|
|
|
|
|
78
|
my @sums = (0) x @$data; |
|
1378
|
14
|
|
|
|
|
42
|
for my $tree ( @{ $self->{trees} } ) { |
|
|
14
|
|
|
|
|
32
|
|
|
1379
|
700
|
|
|
|
|
1047
|
my $root = $tree->{root}; |
|
1380
|
700
|
50
|
|
|
|
965
|
next unless defined $root; |
|
1381
|
700
|
|
|
|
|
955
|
for my $i ( 0 .. $#$data ) { |
|
1382
|
28600
|
|
|
|
|
38660
|
$sums[$i] += $self->_depth_of( $data->[$i], $root ); |
|
1383
|
|
|
|
|
|
|
} |
|
1384
|
|
|
|
|
|
|
} |
|
1385
|
14
|
|
|
|
|
34
|
return \@sums; |
|
1386
|
|
|
|
|
|
|
} ## end sub _depth_sums |
|
1387
|
|
|
|
|
|
|
|
|
1388
|
|
|
|
|
|
|
# Single-row score against the current model state; used by the |
|
1389
|
|
|
|
|
|
|
# prequential score_learn loop, where the normaliser moves as points are |
|
1390
|
|
|
|
|
|
|
# learned and so must be recomputed per row. |
|
1391
|
|
|
|
|
|
|
sub _score_row { |
|
1392
|
1924
|
|
|
1924
|
|
2794
|
my ( $self, $r ) = @_; |
|
1393
|
1924
|
100
|
|
|
|
3261
|
if ( _HAS_ONLINE_XS && $self->{_use_c} ) { |
|
1394
|
|
|
|
|
|
|
|
|
1395
|
|
|
|
|
|
|
# Walks the live trees in C -- no packed snapshot involved, so |
|
1396
|
|
|
|
|
|
|
# this stays fast even though score_learn mutates the trees |
|
1397
|
|
|
|
|
|
|
# between rows. |
|
1398
|
|
|
|
|
|
|
my $sum = Algorithm::Classifier::IsolationForest::online_score_row_xs( $self->{trees}, $r, |
|
1399
|
1324
|
|
|
|
|
8823
|
$self->{n_features}, $self->{max_leaf_samples} ); |
|
1400
|
1324
|
|
|
|
|
2327
|
return exp( -$sum * $self->_score_inv ); |
|
1401
|
|
|
|
|
|
|
} |
|
1402
|
600
|
|
|
|
|
823
|
my $sum = 0; |
|
1403
|
600
|
|
|
|
|
704
|
for my $tree ( @{ $self->{trees} } ) { |
|
|
600
|
|
|
|
|
922
|
|
|
1404
|
10500
|
100
|
|
|
|
20627
|
$sum += $self->_depth_of( $r, $tree->{root} ) if defined $tree->{root}; |
|
1405
|
|
|
|
|
|
|
} |
|
1406
|
600
|
|
|
|
|
1235
|
return exp( -$sum * $self->_score_inv ); |
|
1407
|
|
|
|
|
|
|
} ## end sub _score_row |
|
1408
|
|
|
|
|
|
|
|
|
1409
|
|
|
|
|
|
|
#------------------------------------------------------------------------------- |
|
1410
|
|
|
|
|
|
|
# C-accelerated scoring. |
|
1411
|
|
|
|
|
|
|
# |
|
1412
|
|
|
|
|
|
|
# The parent class's Inline::C scorer walks immutable packed node buffers; |
|
1413
|
|
|
|
|
|
|
# online trees mutate on every learned point. The bridge is a lazily |
|
1414
|
|
|
|
|
|
|
# built snapshot: the first scoring call after any mutation flattens the |
|
1415
|
|
|
|
|
|
|
# live trees into the parent's packed node layout (below) and every |
|
1416
|
|
|
|
|
|
|
# scoring call until the next mutation reuses it. _learn_row -- the one |
|
1417
|
|
|
|
|
|
|
# choke point all mutations flow through -- drops the snapshot. |
|
1418
|
|
|
|
|
|
|
# |
|
1419
|
|
|
|
|
|
|
# Online trees are axis-only, so they map onto the parent's 6-double node |
|
1420
|
|
|
|
|
|
|
# records directly: |
|
1421
|
|
|
|
|
|
|
# |
|
1422
|
|
|
|
|
|
|
# leaf: [0, count, _rpl(count), 0, 0, 0] |
|
1423
|
|
|
|
|
|
|
# axis: [1, attr, split, li, ri, 0] |
|
1424
|
|
|
|
|
|
|
# |
|
1425
|
|
|
|
|
|
|
# The parent packs c(leaf size) into slot 2 and its C walker returns |
|
1426
|
|
|
|
|
|
|
# depth + slot2 at a leaf; packing the online depth-budget adjustment |
|
1427
|
|
|
|
|
|
|
# _rpl(count) there instead makes score_all_xs compute exactly the |
|
1428
|
|
|
|
|
|
|
# pure-Perl _depth_of value, so every downstream C helper (finalize_*, |
|
1429
|
|
|
|
|
|
|
# predict_sums_xs, score_predict_*) applies unchanged. The per-tree |
|
1430
|
|
|
|
|
|
|
# coefficient buffers are empty -- there are no oblique nodes -- and only |
|
1431
|
|
|
|
|
|
|
# exist because score_all_xs expects them. |
|
1432
|
|
|
|
|
|
|
# |
|
1433
|
|
|
|
|
|
|
# score_learn deliberately never uses this path: it mutates the trees |
|
1434
|
|
|
|
|
|
|
# after every single point, so the snapshot could never be reused and |
|
1435
|
|
|
|
|
|
|
# repacking per point would cost more than the walks it replaces. |
|
1436
|
|
|
|
|
|
|
#------------------------------------------------------------------------------- |
|
1437
|
|
|
|
|
|
|
|
|
1438
|
|
|
|
|
|
|
# Drop the packed snapshot; called on every mutation. |
|
1439
|
|
|
|
|
|
|
sub _invalidate_c_trees { |
|
1440
|
16415
|
|
|
16415
|
|
19420
|
delete @{ $_[0] }{qw(_c_nodes _c_coef_idx _c_coef_val)}; |
|
|
16415
|
|
|
|
|
32630
|
|
|
1441
|
16415
|
|
|
|
|
19814
|
return; |
|
1442
|
|
|
|
|
|
|
} |
|
1443
|
|
|
|
|
|
|
|
|
1444
|
|
|
|
|
|
|
# Build (or reuse) the packed snapshot. Returns true when the C scoring |
|
1445
|
|
|
|
|
|
|
# path may be taken, false when the caller must use the pure-Perl walk. |
|
1446
|
|
|
|
|
|
|
sub _ensure_c_trees { |
|
1447
|
92
|
|
|
92
|
|
202
|
my ($self) = @_; |
|
1448
|
92
|
100
|
|
|
|
523
|
return 0 unless $self->{_use_c}; |
|
1449
|
71
|
100
|
|
|
|
353
|
return 1 if $self->{_c_nodes}; |
|
1450
|
|
|
|
|
|
|
|
|
1451
|
27
|
|
|
|
|
53
|
my ( @c_nodes, @c_coef_idx, @c_coef_val ); |
|
1452
|
27
|
|
|
|
|
51
|
my $empty_idx = pack('l*'); |
|
1453
|
27
|
|
|
|
|
52
|
my $empty_val = pack('d*'); |
|
1454
|
27
|
|
|
|
|
40
|
for my $tree ( @{ $self->{trees} } ) { |
|
|
27
|
|
|
|
|
74
|
|
|
1455
|
905
|
|
|
|
|
1551
|
push @c_nodes, $self->_pack_online_tree( $tree->{root} ); |
|
1456
|
905
|
|
|
|
|
1371
|
push @c_coef_idx, $empty_idx; |
|
1457
|
905
|
|
|
|
|
1472
|
push @c_coef_val, $empty_val; |
|
1458
|
|
|
|
|
|
|
} |
|
1459
|
27
|
|
|
|
|
101
|
$self->{_c_nodes} = \@c_nodes; |
|
1460
|
27
|
|
|
|
|
149
|
$self->{_c_coef_idx} = \@c_coef_idx; |
|
1461
|
27
|
|
|
|
|
85
|
$self->{_c_coef_val} = \@c_coef_val; |
|
1462
|
27
|
|
|
|
|
104
|
return 1; |
|
1463
|
|
|
|
|
|
|
} ## end sub _ensure_c_trees |
|
1464
|
|
|
|
|
|
|
|
|
1465
|
|
|
|
|
|
|
# Flatten one live tree into the parent's packed node buffer (DFS |
|
1466
|
|
|
|
|
|
|
# pre-order, root at index 0 -- the origin score_all_xs walks from). |
|
1467
|
|
|
|
|
|
|
sub _pack_online_tree { |
|
1468
|
905
|
|
|
905
|
|
1278
|
my ( $self, $root ) = @_; |
|
1469
|
|
|
|
|
|
|
|
|
1470
|
|
|
|
|
|
|
# A tree that has not learned anything walks as depth 0 with a zero |
|
1471
|
|
|
|
|
|
|
# adjustment: one empty leaf record. |
|
1472
|
905
|
50
|
|
|
|
1430
|
return pack( 'd*', 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 ) unless defined $root; |
|
1473
|
|
|
|
|
|
|
|
|
1474
|
905
|
|
|
|
|
1143
|
my @node_data; |
|
1475
|
|
|
|
|
|
|
my $assign; |
|
1476
|
|
|
|
|
|
|
$assign = sub { |
|
1477
|
7227
|
|
|
7227
|
|
8987
|
my ($node) = @_; |
|
1478
|
7227
|
|
|
|
|
8275
|
my $my_idx = scalar @node_data; |
|
1479
|
7227
|
|
|
|
|
9088
|
push @node_data, undef; # reserve slot; filled in after children |
|
1480
|
7227
|
100
|
|
|
|
10031
|
if ( $node->[_N_TYPE] == _NT_LEAF ) { |
|
1481
|
4066
|
|
|
|
|
5990
|
$node_data[$my_idx] |
|
1482
|
|
|
|
|
|
|
= [ 0.0, $node->[_N_COUNT] + 0.0, $self->_rpl( $node->[_N_COUNT] ) + 0.0, 0.0, 0.0, 0.0 ]; |
|
1483
|
|
|
|
|
|
|
} else { |
|
1484
|
3161
|
|
|
|
|
7428
|
my $li = $assign->( $node->[_N_LEFT] ); |
|
1485
|
3161
|
|
|
|
|
4168
|
my $ri = $assign->( $node->[_N_RIGHT] ); |
|
1486
|
3161
|
|
|
|
|
5949
|
$node_data[$my_idx] |
|
1487
|
|
|
|
|
|
|
= [ 1.0, $node->[_N_ATTR] + 0.0, $node->[_N_SPLIT] + 0.0, $li + 0.0, $ri + 0.0, 0.0 ]; |
|
1488
|
|
|
|
|
|
|
} |
|
1489
|
7227
|
|
|
|
|
9184
|
return $my_idx; |
|
1490
|
905
|
|
|
|
|
2951
|
}; ## end $assign = sub |
|
1491
|
905
|
|
|
|
|
1779
|
$assign->($root); |
|
1492
|
905
|
|
|
|
|
1284
|
return pack( 'd*', map { @$_ } @node_data ); |
|
|
7227
|
|
|
|
|
15242
|
|
|
1493
|
|
|
|
|
|
|
} ## end sub _pack_online_tree |
|
1494
|
|
|
|
|
|
|
|
|
1495
|
|
|
|
|
|
|
# Pack the query rows into the row-major double buffer score_all_xs |
|
1496
|
|
|
|
|
|
|
# reads, via the parent's C row walker. miss_mode 0 maps an undef cell |
|
1497
|
|
|
|
|
|
|
# to 0.0, matching the pure-Perl walk's "// 0". |
|
1498
|
|
|
|
|
|
|
sub _pack_input { |
|
1499
|
71
|
|
|
71
|
|
134
|
my ( $self, $data ) = @_; |
|
1500
|
71
|
|
|
|
|
142
|
my $n_pts = scalar @$data; |
|
1501
|
71
|
|
|
|
|
157
|
my $nf = $self->{n_features}; |
|
1502
|
71
|
|
|
|
|
292
|
my $x_packed = "\0" x ( $n_pts * $nf * 8 ); |
|
1503
|
71
|
|
|
|
|
744
|
Algorithm::Classifier::IsolationForest::pack_input_xs( $data, $x_packed, $n_pts, $nf, 0, '' ); |
|
1504
|
71
|
|
|
|
|
195
|
return ( $n_pts, $x_packed ); |
|
1505
|
|
|
|
|
|
|
} |
|
1506
|
|
|
|
|
|
|
|
|
1507
|
|
|
|
|
|
|
# Lazily learn the contamination threshold from the current window the |
|
1508
|
|
|
|
|
|
|
# first time a predict-family method needs it. A model with no retained |
|
1509
|
|
|
|
|
|
|
# window (window_size 0) stays on the 0.5 fallback until the caller runs |
|
1510
|
|
|
|
|
|
|
# relearn_threshold with data. |
|
1511
|
|
|
|
|
|
|
sub _ensure_threshold { |
|
1512
|
23
|
|
|
23
|
|
84
|
my ($self) = @_; |
|
1513
|
|
|
|
|
|
|
return |
|
1514
|
|
|
|
|
|
|
if !defined $self->{contamination} |
|
1515
|
|
|
|
|
|
|
|| defined $self->{threshold} |
|
1516
|
23
|
100
|
100
|
|
|
117
|
|| !@{ $self->{window} }; |
|
|
4
|
|
66
|
|
|
16
|
|
|
1517
|
4
|
|
|
|
|
22
|
$self->relearn_threshold; |
|
1518
|
4
|
|
|
|
|
11
|
return; |
|
1519
|
|
|
|
|
|
|
} |
|
1520
|
|
|
|
|
|
|
|
|
1521
|
|
|
|
|
|
|
1; |