blib/lib/WWW/Wappalyzer.pm

filter: Statementstmt 99.5% 216 / 217 Branchbran 88.3% 99 / 112 Conditioncond 88.8% 40 / 45 MC/DCmcdc 72.0% 18 / 25 Subroutinesub 100% 20 / 20 Podpod 100% 6 / 6 totaltotal 93.8% 399 / 425 CC 86 SCAR 32.9
CC
86
Cov
95%
detect
49.7
_process_app_techs
14.0
_load_categories_and_techs
9.1
CRAP 26.8
↑ Summary
1 package WWW::Wappalyzer;
2  
3 3 ▶use 5.006;
Subroutine BEGIN: called CC=1 SCAR=0.0
4 3 ▶use strict;
Subroutine BEGIN: called CC=1 SCAR=0.0
5 3 ▶use warnings;
Subroutine BEGIN: called CC=1 SCAR=0.0
6  
7 3 ▶use lib::abs;
Subroutine BEGIN: called CC=1 SCAR=0.0
8 3 ▶use JSON qw();
Subroutine BEGIN: called CC=1 SCAR=0.0
9 3 ▶use Regexp::Parser;
Subroutine BEGIN: called CC=1 SCAR=0.0
10  
11 # List of multi per-page application categories names
12 3 my %MULTIPLE_APP_CATS = map { $_ => 1 } (
13 'Widgets',
14 'Analytics',
15 'JavaScript frameworks',
16 'JavaScript libraries',
17 'UI frameworks',
18 'Video players',
19 'Font scripts',
20 'Miscellaneous',
21 'Advertising',
22 'Payment processors',
23 'JavaScript graphics',
24 'Marketing automation',
25 'Web server extensions',
26 'WordPress plugins',
27 );
28  
29 =head1 NAME
30  
31 WWW::Wappalyzer - Perl port of Wappalyzer (https://wappalyzer.com)
32  
33 =head1 DESCRIPTION
34  
35 Uncovers the technologies used on websites: detects content management systems, web shops,
36 web servers, JavaScript frameworks, analytics tools and many more.
37  
38 Supports only `scriptSrc`, `scripts`, `html`, `meta`, `headers`, 'cookies' and `url` patterns of
39 Wappalyzer specification. Lacks 'version', 'implies', 'excludes' support in favour of speed.
40  
41 Categories: L<https://github.com/wappalyzer/wappalyzer/blob/master/src/categories.json>
42 Technologies: L<https://github.com/wappalyzer/wappalyzer/tree/master/src/technologies>
43 More info on Wappalyzer: L<https://github.com/wappalyzer/wappalyzer>
44  
45 =cut
46  
47 3 our $VERSION = '2.00';
48  
49 =head1 SYNOPSIS
50  
51 use WWW::Wappalyzer;
52 use LWP::UserAgent;
53 use List::Util 'pairmap';
54  
55 my $response = LWP::UserAgent->new->get( 'http://www.drupal.org' );
56 my %detected = WWW::Wappalyzer->new->detect(
57 html => $response->decoded_content,
58 headers => { pairmap { $a => [ $response->headers->header($a) ] } $response->headers->flatten },
59 );
60  
61 # %detected = (
62 # 'Font scripts' => [ 'Google Font API' ],
63 # 'Caching' => [ 'Varnish' ],
64 # 'CDN' => [ 'Fastly' ],
65 # 'CMS' => [ 'Drupal' ],
66 # 'Video players' => [ 'YouTube' ],
67 # 'Tag managers' => [ 'Google Tag Manager' ],
68 # 'Reverse proxies' => [ 'Nginx' ],
69 # 'Web servers' => [ 'Nginx' ],
70 # );
71  
72 =head1 EXPORT
73  
74 None by default.
75  
76 =head1 SUBROUTINES/METHODS
77  
78 =head2 new
79  
80 my $wappalyzer = WWW::Wappalyzer->new( %params )
81  
82 Constructor.
83  
84 Available parameters:
85  
86 categories - optional additional categories array ref to files list (refer 'add_categories_files' below)
87 technologies - optional additional technologies array ref to files list (refer 'add_technologies_files' below)
88  
89 Returns the instance of WWW::Wappalyzer class.
90  
91 =cut
92  
93 2 ▶sub new {
Subroutine new: called CC=3 SCAR=11.0
Pod documented
94 2 my ( $class, %params ) = @_;
95  
96 2 my $self = bless {
97 _categories => {},
98 _cats_file_list => [],
99 _techs_file_list => [],
100 }, $class;
101  
102 2 $self->add_categories_files( lib::abs::path( './wappalyzer_src/categories.json' ) );
103 2 $self->add_technologies_files( glob lib::abs::path( './wappalyzer_src/technologies' ) . '/*.json' );
104  
105 2 ▶ if ( ref $params{categories} eq 'ARRAY' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $params{"categories"} eq "ARRAY") 1 1 2
106 1 $self->add_categories_files( @{ $params{categories} } );
107 }
108  
109 2 ▶ if ( ref $params{technologies} eq 'ARRAY' ) {
Branch: 1 branch
BranchTrueFalseTotal
if (ref $params{"technologies"} eq "ARRAY") 1 1 2
110 1 $self->add_technologies_files( @{ $params{technologies} } );
111 }
112  
113 2 return $self;
114 }
115  
116 =head2 detect
117  
118 my %detected = $wappalyzer->detect( %params )
119  
120 Tries to detect CMS, framework, etc for given html code, http headers, URL.
121  
122 Available parameters:
123  
124 html - HTML code of web page.
125  
126 headers - Hash ref to http headers list. The value may be a plain string or an array ref
127 of strings for a multi-valued field.
128 Cookies should be passed in 'Set-Cookie' header.
129  
130 url - URL of web page.
131  
132 cats - Array ref to a list of trying categories names, defaults to all.
133 Less categories - less CPU usage.
134  
135 Returns the hash of detected applications by category:
136  
137 (
138 CMS => [ 'Joomla' ],
139 'Javascript frameworks' => [ 'jQuery', 'jQuery UI' ],
140 )
141  
142 =cut
143  
144 18 ▶sub detect {
Subroutine detect: called CC=49 SCAR=39.1
Pod documented
145 18 my ( $self, %params ) = @_;
146  
147 18 ▶ return () unless $params{html} || $params{headers} || $params{url};
Branch: 1 branch
BranchTrueFalseTotal
unless $params{"html"} or $params{"headers"} or $params{"url"} 0 18 18
Condition83%5/6
$params{"html"} or $params{"headers"} or $params{"url"}
l!l&&r!l&&!r
1710
$params{"html"} or $params{"headers"}
l!l&&r!l&&!r
1161
MC/DC0%0/3
$params{"html"} or $params{"headers"} or $params{"url"}
$params{"html"} $params{"headers"} $params{"url"}
Truth table75%3/4
A or B or C
A= $params{"html"}B= $params{"headers"}C= $params{"url"}
ABCresult
1XX1
01X1
0011
0000
148
149 # search will be case insensitive
150 18 ▶ if ( $params{html} ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($params{"html"}) 11 7 18
151 11 $params{html} = lc $params{html};
152 }
153  
154 18 ▶ if ( $params{url} ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($params{"url"}) 1 17 18
155 1 $params{url} = lc $params{url};
156 }
157  
158 18 my $headers_ref;
159 18 ▶ if ( $params{headers} ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($params{"headers"}) 14 4 18
160 14 ▶ die 'Bad headers param' unless ref $params{headers} eq 'HASH';
Branch: 1 branch
BranchTrueFalseTotal
unless ref $params{"headers"} eq "HASH" 1 13 14
161  
162 # Make all headers lowercase and array ref valued
163 13 $headers_ref = {};
164 13 ▶ while ( my ( $header, $header_vals_ref ) = each %{ $params{headers} } ) {
Branch: 1 branch
BranchTrueFalseTotal
if (() = each %{$params{"headers"};}) 14 13 27
165 14 ▶ unless ( ref $header_vals_ref ) {
Branch: 2 branches
BranchTrueFalseTotal
if (ref $header_vals_ref) { } 5 9 14
if (do { ref $header_vals_ref ne "ARRAY" }) 1 4 5
166 9 $header_vals_ref = [ $header_vals_ref ];
167 }
168 0 elsif ( ref $header_vals_ref ne 'ARRAY' ) {
169 1 next;
170 }
171  
172 13 $headers_ref->{ lc $header } = [ map { lc } @$header_vals_ref ];
173 }
174 }
175  
176 # Lazy load and process techs from JSON file
177 17 ▶ $self->_load_categories_and_techs unless scalar keys %{ $self->{_categories} };
Branch: 1 branch
BranchTrueFalseTotal
unless scalar keys %{$$self{"_categories"};} 3 14 17
178  
179 my @cats = $params{cats} && ( ref( $params{cats} ) || '' ) eq 'ARRAY'
180 17 ▶ ? @{ $params{cats} } : $self->get_categories_names;
Branch: 1 branch
BranchTrueFalseTotal
$params{'cats'} && (ref $params{'cats'} || '') eq 'ARRAY' ? : 2 15 17
Condition60%3/5
$params{"cats"} and (ref $params{"cats"} || "") eq "ARRAY"
!ll&&!rl&&r
1502
ref $params{'cats'} || ''
l!l
20
MC/DC50%1/2
$params{"cats"} and (ref $params{"cats"} || "") eq "ARRAY"
$params{"cats"} ref $params{'cats'}
Truth table66%2/3
A and B
A= $params{"cats"}B= ref $params{'cats'}
ABresult
0X0
100
111
181  
182  
183 17 my %cookies;
184 17 ▶ if ( my $cookies_header = $headers_ref->{'set-cookie'} ) {
Branch: 1 branch
BranchTrueFalseTotal
if (my $cookies_header = $headers_ref->{"set-cookie"}) 3 14 17
185 3 for my $cookie_str ( @$cookies_header ) {
186 5 ▶ next unless $cookie_str =~ /^(?<name>.+?)=(?<value>.*?);\s/;
Branch: 1 branch
BranchTrueFalseTotal
unless $cookie_str =~ /^(?<name>.+?)=(?<value>.*?);\s/ 0 5 5
187 5 $cookies{ lc $+{name} } = $+{value};
188 }
189 }
190  
191 17 my %detected;
192 my %tried_multi_cat_apps;
193 17 for my $cat ( @cats ) {
194 1469 ▶ my $apps_ref = $self->{_categories}{ $cat } or die "Unknown category name $cat";
Branch: 1 branch
BranchTrueFalseTotal
my $apps_ref = $self->{"_categories"}{$cat} or die "Unknown category name $cat" 1469 0 1469
195  
196 APP:
197 1469 for my $app_ref ( @$apps_ref ) {
198  
199 52605 my $detected;
200  
201 # Some speed optimizations
202 52605 ▶ if ( @cats > 1 && $app_ref->{multi_cat}
Branch: 1 branch
BranchTrueFalseTotal
if (@cats > 1 and $app_ref->{'multi_cat'} and exists $tried_multi_cat_apps{$app_ref->{'name'}}) { } 8036 44569 52605
Condition100%6/6
@cats > 1 and $app_ref->{"multi_cat"} and exists $tried_multi_cat_apps{$app_ref->{"name"}}
!ll&&!rl&&r
3700975608036
@cats > 1 and $app_ref->{"multi_cat"}
!ll&&!rl&&r
843692515596
MC/DC100%3/3
@cats > 1 and $app_ref->{"multi_cat"} and exists $tried_multi_cat_apps{$app_ref->{"name"}}
@cats > 1 $app_ref->{"multi_cat"} exists $tried_multi_cat_apps{$app_ref->{"name"}}
Truth table100%4/4
A and B and C
A= @cats > 1B= $app_ref->{"multi_cat"}C= exists $tried_multi_cat_apps{$app_ref->{"name"}}
ABCresult
0XX0
10X0
1100
1111
203 && exists $tried_multi_cat_apps{ $app_ref->{name} }
204 ) {
205 8036 $detected = $tried_multi_cat_apps{ $app_ref->{name} };
206 }
207 else {
208 # Try regexes...
209 44569 my $confidence = 0;
210  
211 44569 ▶ if ( defined $headers_ref && exists $app_ref->{headers_rules} ) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $headers_ref and exists $app_ref->{"headers_rules"}) 5987 38582 44569
Condition66%2/3
defined $headers_ref and exists $app_ref->{"headers_rules"}
!ll&&!rl&&r
0385825987
MC/DC50%1/2
defined $headers_ref and exists $app_ref->{"headers_rules"}
defined $headers_ref exists $app_ref->{"headers_rules"}
Truth table66%2/3
A and B
A= defined $headers_refB= exists $app_ref->{"headers_rules"}
ABresult
0X0
100
111
212 5987 my %headers_rules = %{ $app_ref->{headers_rules} };
213  
214 HEADER_RULE:
215 5987 ▶ while ( my ( $header, $rule ) = each %headers_rules ) {
Branch: 1 branch
BranchTrueFalseTotal
if (() = each %headers_rules) 8096 5977 14073
216 8096 ▶ my $header_vals_ref = $headers_ref->{ $header } or next;
Branch: 1 branch
BranchTrueFalseTotal
my $header_vals_ref = $headers_ref->{$header} or next 906 7190 8096
217  
218 906 for my $header_val ( @$header_vals_ref ) {
219 910 ▶ if ( $header_val =~ m/$rule->{re}/ ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($header_val =~ /$rule->{"re"}/) 10 900 910
220 10 $confidence += $rule->{confidence};
221 10 ▶ if ( $confidence >= 100 ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($confidence >= 100) 10 0 10
222 10 $detected = 1;
223 10 last HEADER_RULE;
224 }
225 }
226 }
227 }
228 }
229  
230 44569 ▶ if ( !$detected && exists $app_ref->{cookies_rules} && scalar keys %cookies ) {
Branch: 1 branch
BranchTrueFalseTotal
if (not $detected and exists $app_ref->{"cookies_rules"} and scalar keys %cookies) 576 43993 44569
Condition100%5/5
not $detected and exists $app_ref->{"cookies_rules"} and scalar keys %cookies
!ll
417232846
not $detected and exists $app_ref->{"cookies_rules"}
!ll&&!rl&&r
10417132846
MC/DC100%2/2
not $detected and exists $app_ref->{"cookies_rules"} and scalar keys %cookies
not $detected exists $app_ref->{"cookies_rules"}
Truth table100%3/3
not(A) and B
A= not $detectedB= exists $app_ref->{"cookies_rules"}
ABresult
0X0
100
111
231 576 my %cookies_rules = %{ $app_ref->{cookies_rules} };
232  
233 COOKIE_RULE:
234 576 ▶ while ( my ( $cookie, $rule ) = each %cookies_rules ) {
Branch: 1 branch
BranchTrueFalseTotal
if (() = each %cookies_rules) 843 573 1416
235 843 ▶ my $cookie_val = $cookies{ $cookie } or next;
Branch: 1 branch
BranchTrueFalseTotal
my $cookie_val = $cookies{$cookie} or next 3 840 843
236  
237 3 ▶ if ( $cookie_val =~ /$rule->{re}/ ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($cookie_val =~ /$rule->{"re"}/) 3 0 3
238 3 $confidence += $rule->{confidence};
239 3 ▶ if ( $confidence >= 100 ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($confidence >= 100) 3 0 3
240 3 $detected = 1;
241 3 last COOKIE_RULE;
242 }
243 }
244 }
245 }
246  
247 44569 ▶ unless ( $detected ) {
Branch: 1 branch
BranchTrueFalseTotal
unless ($detected) 44556 13 44569
248 # try from most to least relevant method
249 RULES:
250 44556 for my $rule_type ( qw( html url ) ) {
251 89104 my $rule_name = $rule_type . '_rules';
252 89104 ▶ if ( defined $params{ $rule_type } && exists $app_ref->{ $rule_name } ) {
Branch: 1 branch
BranchTrueFalseTotal
if (defined $params{$rule_type} and exists $app_ref->{$rule_name}) 18317 70787 89104
Condition100%3/3
defined $params{$rule_type} and exists $app_ref->{$rule_name}
!ll&&!rl&&r
595141127318317
MC/DC100%2/2
defined $params{$rule_type} and exists $app_ref->{$rule_name}
defined $params{$rule_type} exists $app_ref->{$rule_name}
Truth table100%3/3
A and B
A= defined $params{$rule_type}B= exists $app_ref->{$rule_name}
ABresult
0X0
100
111
253 18317 for my $rule ( @{ $app_ref->{ $rule_name } } ) {
254 18872 ▶ if ( $params{ $rule_type } =~ m/$rule->{re}/ ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($params{$rule_type} =~ /$rule->{"re"}/) 11 18861 18872
255 11 $confidence += $rule->{confidence};
256 11 ▶ if ( $confidence >= 100 ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($confidence >= 100) 9 2 11
257 9 $detected = 1;
258 9 last RULES;
259 }
260 }
261 }
262 }
263 }
264 }
265  
266 # Some speed optimizations
267 44569 ▶ if ( @cats > 1 && $app_ref->{multi_cat} ) {
Branch: 1 branch
BranchTrueFalseTotal
if (@cats > 1 and $app_ref->{"multi_cat"}) 7560 37009 44569
Condition100%3/3
@cats > 1 and $app_ref->{"multi_cat"}
!ll&&!rl&&r
84369257560
MC/DC100%2/2
@cats > 1 and $app_ref->{"multi_cat"}
@cats > 1 $app_ref->{"multi_cat"}
Truth table100%3/3
A and B
A= @cats > 1B= $app_ref->{"multi_cat"}
ABresult
0X0
100
111
268 7560 $tried_multi_cat_apps{ $app_ref->{name} } = $detected;
269 }
270 }
271  
272 52605 ▶ next unless $detected;
Branch: 1 branch
BranchTrueFalseTotal
unless $detected 52577 28 52605
273  
274 # Detected!
275 28 push @{ $detected{ $cat } }, $app_ref->{name};
276  
277 28 ▶ last APP unless $MULTIPLE_APP_CATS{ $cat };
Branch: 1 branch
BranchTrueFalseTotal
unless $MULTIPLE_APP_CATS{$cat} 26 2 28
278 }
279 }
280  
281 17 return %detected;
282 }
283  
284 =head2 get_categories_names
285  
286 my @cats = $wappalyzer->get_categories_names()
287  
288 Returns the array of all application categories names.
289  
290 =cut
291  
292 16 ▶sub get_categories_names {
Subroutine get_categories_names: called CC=2 SCAR=6.9
Pod documented
293 16 my ( $self ) = @_;
294  
295 # Lazy load and process categories from JSON files
296 16 ▶ $self->_load_categories_and_techs() unless scalar keys %{ $self->{_categories} };
Branch: 1 branch
BranchTrueFalseTotal
unless scalar keys %{$$self{"_categories"};} 1 15 16
297  
298 16 return keys %{ $self->{_categories} };
299 }
300  
301 # Loads and processes categories and techs from JSON files
302 4 ▶sub _load_categories_and_techs {
Subroutine _load_categories_and_techs: called CC=9 SCAR=22.1
303 4 my ( $self ) = @_;
304  
305 4 my $cats_ref = {};
306  
307 4 for my $cats_file ( @{ $self->{_cats_file_list} } ) {
308 7 $cats_ref = { %$cats_ref, %{ _load_json( $cats_file ) } };
309 }
310  
311 4 for my $techs_file ( @{ $self->{_techs_file_list} } ) {
312 111 my $apps_ref = _load_json( $techs_file );
313  
314 # Process apps
315 111 ▶ while ( my ( $app, $app_ref ) = each %$apps_ref ) {
Branch: 1 branch
BranchTrueFalseTotal
if (() = each %$apps_ref) 11941 111 12052
316  
317 11941 ▶ my $new_app_ref = _process_app_techs( $app, $app_ref ) or next;
Branch: 1 branch
BranchTrueFalseTotal
my $new_app_ref = _process_app_techs($app, $app_ref) or next 11941 0 11941
318  
319 11941 ▶ my @cats = @{ $app_ref->{cats} } or next;
Branch: 1 branch
BranchTrueFalseTotal
my(@cats) = @{$app_ref->{"cats"};} or next 11941 0 11941
320  
321 11941 ▶ $new_app_ref->{multi_cat} = 1 if @cats > 1;
Branch: 1 branch
BranchTrueFalseTotal
if @cats > 1 2016 9925 11941
322  
323 11941 for my $cat_id ( @cats ) {
324 14101 ▶ my $cat = $cats_ref->{ $cat_id } or die "Bad categorie id $cat_id in app $app";
Branch: 1 branch
BranchTrueFalseTotal
my $cat = $cats_ref->{$cat_id} or die "Bad categorie id $cat_id in app $app" 14101 0 14101
325  
326 14101 push @{ $self->{_categories}{ $cat->{name} } }, $new_app_ref;
327 }
328 }
329 }
330 }
331  
332 # Loads JSON file
333 118 ▶sub _load_json {
Subroutine _load_json: called CC=4 SCAR=14.0
334 118 my ( $file ) = @_;
335  
336 118 ▶ open my $fh, '<', $file or die "Can not read file $file.";
Branch: 1 branch
BranchTrueFalseTotal
open my $fh, "<", $file or die "Can not read file $file." 118 0 118
337  
338 118 local $/ = undef;
339 118 my $json = <$fh>;
340 118 close $fh;
341  
342 # Replace html entities with oridinary symbols
343 118 $json =~ s{&gt;}{>}xig;
344 118 $json =~ s{&lt;}{<}xig;
345  
346 118 my $res = eval { JSON::decode_json( $json ) };
347  
348 118 ▶ die "Can't parse JSON file $file: $@" if $@;
Branch: 1 branch
BranchTrueFalseTotal
if $@ 0 118 118
349  
350 118 ▶ die "$file has invalid format" unless ref $res eq 'HASH';
Branch: 1 branch
BranchTrueFalseTotal
unless ref $res eq "HASH" 0 118 118
351  
352 118 return $res;
353 }
354  
355 # Process techs of given app
356 11941 ▶sub _process_app_techs {
Subroutine _process_app_techs: called CC=14 SCAR=26.4
357 11941 my ( $app, $app_ref ) = @_;
358  
359 11941 my $new_app_ref = { name => $app };
360  
361 11941 my @fields = grep { exists $app_ref->{ $_ } }
362 qw( scriptSrc scripts html meta headers cookies url );
363  
364 11941 my @html_rules;
365  
366 # Precompile regexps
367 11941 for my $field ( @fields ) {
368 11895 my $rule_ref = $app_ref->{ $field };
369 my @rules_list = !ref $rule_ref ? _parse_rule( $rule_ref )
370 11895 ▶ : ref $rule_ref eq 'ARRAY' ? ( map { _parse_rule( $_ ) } @$rule_ref )
Branch: 2 branches
BranchTrueFalseTotal
!ref($rule_ref) ? : 6864 5031 11895
ref $rule_ref eq 'ARRAY' ? : 1228 3803 5031
371 : () ;
372  
373 11895 ▶ if ( $field eq 'html' || $field eq 'scripts' ) {
Branch: 5 branches
BranchTrueFalseTotal
if ($field eq 'html' or $field eq 'scripts') { } 1449 10446 11895
elsif ($field eq 'scriptSrc') { } 6371 4075 10446
elsif ($field eq 'url') { } 272 3803 4075
elsif ($field eq 'meta') { } 1403 2400 3803
elsif ($field eq 'headers' or $field eq 'cookies') { } 2400 0 2400
Condition100%3/3
$field eq "html" or $field eq "scripts"
l!l&&r!l&&!r
13579210446
MC/DC100%2/2
$field eq "html" or $field eq "scripts"
$field eq "html" $field eq "scripts"
Truth table100%3/3
A or B
A= $field eq "html"B= $field eq "scripts"
ABresult
1X1
011
000
374 1449 push @html_rules, map { $_->{re} = qr/(?-x:$_->{re})/; $_ } @rules_list;
375 }
376 elsif ( $field eq 'scriptSrc' ) {
377 push @html_rules,
378 map {
379 6371 $_->{re} = qr/
380 < \s* script [^>]+ src \s* = \s* ["'] [^"']* (?-x:$_->{re}) [^"']* ["']
381 /x;
382 7163 $_
383 } @rules_list;
384 }
385 elsif ( $field eq 'url' ) {
386 272 my @url_rules = map { $_->{re} = qr/(?-x:$_->{re})/; $_ } @rules_list;
387 272 $new_app_ref->{url_rules} = _optimize_rules( \@url_rules );
388 }
389 elsif ( $field eq 'meta' ) {
390 1403 for my $key ( keys %$rule_ref ) {
391 1583 my $lc_key = lc $key;
392 1583 my $name_re = qr/ name \s* = \s* ["']? $lc_key ["']? /x;
393 1583 my $rule = _parse_rule( $rule_ref->{ $key } );
394 1583 $rule->{re} = qr/$rule->{re}/;
395 1583 my $content_re = qr/ content \s* = \s* ["'] [^"']* (?-x:$rule->{re}) [^"']* ["'] /x;
396  
397 1583 $rule->{re} = qr/
398 < \s* meta \s+
399 (?:
400 (?: $name_re \s+ $content_re )
401 # | (?: $content_re \s+ $name_re ) # hangs sometimes
402 )
403 /x;
404
405 1583 push @html_rules, $rule;
406 }
407 }
408 ▶ elsif ( $field eq 'headers' || $field eq 'cookies' ) {
Condition66%2/3
$field eq "headers" or $field eq "cookies"
l!l&&r!l&&!r
16277730
MC/DC0%0/2
$field eq "headers" or $field eq "cookies"
$field eq "headers" $field eq "cookies"
Truth table66%2/3
A or B
A= $field eq "headers"B= $field eq "cookies"
ABresult
1X1
011
000
409 2400 for my $key ( keys %$rule_ref ) {
410 3328 my $rule = _parse_rule( $rule_ref->{ $key } );
411 3328 $rule->{re} = qr/$rule->{re}/;
412 3328 $new_app_ref->{ $field . '_rules' }{ lc $key } = $rule;
413 }
414 }
415 }
416  
417 11941 ▶ if ( @html_rules ) {
Branch: 1 branch
BranchTrueFalseTotal
if (@html_rules) 8149 3792 11941
418 8149 $new_app_ref->{html_rules} = _optimize_rules( \@html_rules );
419 }
420  
421 11941 return $new_app_ref;
422 }
423  
424 # separate regexp and other optional parameters from the rule
425 14207 ▶sub _parse_rule {
Subroutine _parse_rule: called CC=5 SCAR=16.1
426 14207 my ( $rule ) = @_;
427
428 14207 my ( $re, @tags ) = split /\\;/, $rule;
429
430 14207 my $confidence;
431 14207 for my $tag ( @tags ) {
432 2658 ▶ if ( ( $confidence ) = $tag =~ /^\s*confidence\s*:\s*(\d+)\s*$/ ) {
Branch: 1 branch
BranchTrueFalseTotal
if (($confidence) = $tag =~ /^\s*confidence\s*:\s*(\d+)\s*$/) 266 2392 2658
433 # supports only confidence for now
434 266 last;
435 }
436 }
437
438 return {
439 14207 ▶ re => _escape_re( defined( $re ) ? $re : '' ),
Branch: 1 branch
BranchTrueFalseTotal
defined $re ? : 12464 1743 14207
Condition100%2/2
$confidence || 100
l!l
25413953
MC/DC100%1/1
$confidence || 100
$confidence
Truth table100%2/2
A
A= $confidence
Aresult
11
00
440 confidence => $confidence || 100,
441 };
442 }
443  
444 # Escape special symbols in regexp string of config file
445 14207 ▶sub _escape_re {
Subroutine _escape_re: called CC=5 SCAR=16.1
446 14207 my ( $re ) = @_;
447
448 # Escape { } braces
449 #$re =~ s/ ([{}]) /[$1]/xig;
450  
451 # Escape [^]
452 14207 $re =~ s{\Q[^]\E}{[\\^]}ig;
453  
454 # Escape \\1
455 14207 $re =~ s{\Q\1\E}{\\\\1}ig;
456  
457 # Escape (?!
458 14207 $re =~ s{[(][?][!]}{([?]!}ig;
459
460 # turn literals in regexp to lowercase to make case insensitive search
461 # i flag will be slower because we makes many searches in one text
462 3 ▶ no warnings 'redefine';
Subroutine BEGIN: called CC=1 SCAR=0.0
463 14207 ▶ local *Regexp::Parser::warn = sub {}; # it may be too noisy
Subroutine __ANON__: called CC=1 SCAR=0.0
464
465 14207 my $parser = Regexp::Parser->new();
466 14207 ▶ if ( $parser->regex($re) ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($parser->regex($re)) 12408 1799 14207
467 12408 $re = '';
468
469 12408 ▶ while ( my $node = $parser->next ) {
Branch: 1 branch
BranchTrueFalseTotal
if (my $node = $parser->next) 314175 12408 326583
470 314175 my $ref = ref $node;
471 314175 ▶ if ( $ref eq 'Regexp::Parser::exact' || $ref eq 'Regexp::Parser::anyof_char' ) {
Branch: 1 branch
BranchTrueFalseTotal
if ($ref eq 'Regexp::Parser::exact' or $ref eq 'Regexp::Parser::anyof_char') { } 272253 41922 314175
Condition100%3/3
$ref eq "Regexp::Parser::exact" or $ref eq "Regexp::Parser::anyof_char"
l!l&&r!l&&!r
267975427841922
MC/DC100%2/2
$ref eq "Regexp::Parser::exact" or $ref eq "Regexp::Parser::anyof_char"
$ref eq "Regexp::Parser::exact" $ref eq "Regexp::Parser::anyof_char"
Truth table100%3/3
A or B
A= $ref eq "Regexp::Parser::exact"B= $ref eq "Regexp::Parser::anyof_char"
ABresult
1X1
011
000
472 272253 $re .= lc $node->raw;
473 }
474 else {
475 41922 $re .= $node->raw;
476 }
477 }
478 }
479
480 14207 return $re;
481 }
482  
483 # If possible combine all rules in one regexp
484 8421 ▶sub _optimize_rules {
Subroutine _optimize_rules: called CC=3 SCAR=11.0
485 8421 my ( $rules ) = @_;
486
487 8421 ▶ if ( @$rules > 1 && @$rules == grep { $_->{confidence} == 100 } @$rules ) {
Branch: 1 branch
BranchTrueFalseTotal
if (@$rules > 1 and @$rules == grep({$_->{"confidence"} == 100;} @$rules)) 1575 6846 8421
Condition100%3/3
@$rules > 1 and @$rules == grep({$_->{"confidence"} == 100;} @$rules)
!ll&&!rl&&r
67311151575
MC/DC100%2/2
@$rules > 1 and @$rules == grep({$_->{"confidence"} == 100;} @$rules)
@$rules > 1 @$rules == grep({$_->{"confidence"} == 100;} @$rules)
Truth table100%3/3
A and B
A= @$rules > 1B= @$rules == grep({$_->{"confidence"} == 100;} @$rules)
ABresult
0X0
100
111
488 # can combine only if confidence for each is 100
489 1575 my $re = join '|', map { $_->{re} } @$rules;
490 return [{
491 1575 re => qr/$re/,
492 confidence => 100,
493 }];
494 }
495
496 6846 return $rules;
497 }
498  
499 =head2 add_categories_files
500  
501 $wappalyzer->add_categories_files( @filepaths )
502  
503 Puts additional categories files to a list of processed categories files.
504 See lib/WWW/wappalyzer_src/categories.json as format sample.
505  
506 =cut
507  
508 4 ▶sub add_categories_files {
Subroutine add_categories_files: called CC=1 SCAR=0.0
Pod documented
509 4 my ( $self, @filepaths ) = @_;
510  
511 4 push @{ $self->{_cats_file_list} }, @filepaths;
512  
513 # just clear out categories to lazy load later
514 4 $self->{_categories} = {};
515 }
516  
517 =head2 add_technologies_files
518  
519 $wappalyzer->add_technologies_files( @filepaths )
520  
521 Puts additional techs files to a list of processed techs files.
522 See lib/WWW/wappalyzer_src/technologies/a.json as format sample.
523  
524 =cut
525  
526 4 ▶sub add_technologies_files {
Subroutine add_technologies_files: called CC=1 SCAR=0.0
Pod documented
527 4 my ( $self, @filepaths ) = @_;
528  
529 4 push @{ $self->{_techs_file_list} }, @filepaths;
530  
531 # just clear out categories to lazy load later
532 4 $self->{_categories} = {};
533 }
534  
535 =head2 reload_files
536  
537 $wappalyzer->reload_files()
538  
539 Ask to reload data from additional categories and technologies files
540 those may be changed in runtime.
541  
542 =cut
543  
544 1 ▶sub reload_files {
Subroutine reload_files: called CC=1 SCAR=0.0
Pod documented
545 1 my ( $self ) = @_;
546  
547 # just clear out categories to lazy load later
548 1 $self->{_categories} = {};
549 }
550  
551  
552 =head1 AUTHOR
553  
554 Alexander Nalobin, C<< <alexander at nalobin.ru> >>
555  
556 =head1 BUGS
557  
558 Please report any bugs or feature requests to C<bug-www-wappalyzer at rt.cpan.org>, or through
559 the web interface at L<http://rt.cpan.org/NoAuth/ReportBug.html?Queue=WWW-Wappalyzer>. I will be notified, and then you'll
560 automatically be notified of progress on your bug as I make changes.
561  
562  
563 =head1 SUPPORT
564  
565 You can find documentation for this module with the perldoc command.
566  
567 perldoc WWW::Wappalyzer
568  
569  
570 You can also look for information at:
571  
572 =over 4
573  
574 =item * GitHub
575  
576 L<https://github.com/nalobin/WWW-Wappalyzer>
577  
578 =item * RT: CPAN's request tracker (report bugs here)
579  
580 L<http://rt.cpan.org/NoAuth/Bugs.html?Dist=WWW-Wappalyzer>
581  
582 =item * AnnoCPAN: Annotated CPAN documentation
583  
584 L<http://annocpan.org/dist/WWW-Wappalyzer>
585  
586 =item * CPAN Ratings
587  
588 L<http://cpanratings.perl.org/d/WWW-Wappalyzer>
589  
590 =item * Search CPAN
591  
592 L<http://search.cpan.org/dist/WWW-Wappalyzer/>
593  
594 =back
595  
596  
597 =head1 ACKNOWLEDGEMENTS
598  
599  
600 =head1 LICENSE AND COPYRIGHT
601  
602 Copyright 2013-2015 Alexander Nalobin.
603  
604 This program is free software; you can redistribute it and/or modify it
605 under the terms of either: the GNU General Public License as published
606 by the Free Software Foundation; or the Artistic License.
607  
608 See http://dev.perl.org/licenses/ for more information.
609  
610  
611 =cut
612  
613 3 1; # End of WWW::Wappalyzer
↑ Summary