File Coverage

blib/lib/WWW/Shopify/Liquid/Lexer.pm

Criterion	Covered	Total	%
statement	75	428	17.5
branch	0	250	0.0
condition	0	282	0.0
subroutine	25	70	35.7
pod	0	16	0.0
total	100	1046	9.5

#!/usr/bin/perl

use strict;
use warnings;

package WWW::Shopify::Liquid::Token;
use base 'WWW::Shopify::Liquid::Element';
sub new { return bless { line => $_[1], core => $_[2] }, $_[0]; };
sub stringify { return $_[0]->{core}; }
sub tokens { return $_[0]; }

package WWW::Shopify::Liquid::Token::Operator;
use base 'WWW::Shopify::Liquid::Token';

package WWW::Shopify::Liquid::Token::Operand;
use base 'WWW::Shopify::Liquid::Token';

package WWW::Shopify::Liquid::Token::String;
use base 'WWW::Shopify::Liquid::Token::Operand';
sub process { my ($self, $hash) = @_; return $self->{core}; }

package WWW::Shopify::Liquid::Token::Number;
use base 'WWW::Shopify::Liquid::Token::Operand';
sub process { my ($self, $hash) = @_; return $self->{core}; }

package WWW::Shopify::Liquid::Token::NULL;
use base 'WWW::Shopify::Liquid::Token::Operand';
sub process { return undef; }

package WWW::Shopify::Liquid::Token::Bool;
use base 'WWW::Shopify::Liquid::Token::Operand';
sub process { my ($self, $hash) = @_; return $self->{core}; }

package WWW::Shopify::Liquid::Token::Variable;
use base 'WWW::Shopify::Liquid::Token::Operand';

use Scalar::Util qw(looks_like_number reftype blessed);

sub new { my $package = shift; return bless { line => shift, core => [@_] }, $package; };
sub process {
	my ($self, $hash, $action, $pipeline) = @_;
	my $place = $hash;

	my @inner = @{$self->{core}};
	my $unprocessed = 0;
	foreach my $part_idx (0..$#inner) {
		my $part = $inner[$part_idx];
		if (ref($part) eq 'WWW::Shopify::Liquid::Token::Variable::Processing') {
			$place = $part->$action($pipeline, $hash, $place);
		}
		else {
			my $key = $self->is_processed($part) ? $part : $part->$action($pipeline, $hash);

			return $self unless defined $key && $key ne '';
			$self->{core}->[$part_idx] = $key if $self->is_processed($key) && $action eq "optimize";
			if (defined $place) {
				if (reftype($place) && reftype($place) eq "HASH" && exists $place->{$key}) {
					$place = $place->{$key};
				} elsif (reftype($place) && reftype($place) eq "ARRAY" && looks_like_number($key) && defined $place->[$key]) {
					$place = $place->[$key];
				} elsif ($pipeline->make_method_calls && blessed($place) && $place->can($key)) {
					$place = $place->$key;
				} else {
					$unprocessed = 1;
					$place = undef;
				}
			}

		}
	}
	return $unprocessed ? $self : $place;
}
sub stringify { return join(".", map { $_->stringify } @{$_[0]->{core}}); }

sub set {
	my ($self, $pipeline, $hash, $value) = @_;
	my @vars = map { $self->is_processed($_) ? $_ : $_->render($pipeline, $hash) } @{$self->{core}};
	my ($reference) = $pipeline->variable_reference($hash, \@vars);
	$$reference = $value;
	return 1;
}


sub get {
	my ($self, $pipeline, $hash) = @_;
	my @vars = map { $self->is_processed($_) ? $_ : $_->render($pipeline, $hash) } @{$self->{core}};
	my ($reference) = $pipeline->variable_reference($hash, \@vars, 1);
	return $reference ? $$reference : undef;
}

package WWW::Shopify::Liquid::Token::Variable::Processing;
use base 'WWW::Shopify::Liquid::Token::Operand';
sub process {
	my ($self, $hash, $argument, $action, $pipeline) = @_;
	return $self if !$self->is_processed($argument);
	my $result = $self->{core}->operate($hash, $argument);
	return $self if !$self->is_processed($result);
	return $result;
}

package WWW::Shopify::Liquid::Token::Variable::Named;
use base 'WWW::Shopify::Liquid::Token::Operand';

sub new { my $package = shift; return bless { line => shift, name => shift, core => shift }, $package; };
sub process {
	my ($self, $hash, $action, $pipeline) = @_;
	return { $self->{name} => $self->{core}->$action($pipeline, $hash) };
}


package WWW::Shopify::Liquid::Token::Grouping;
use base 'WWW::Shopify::Liquid::Token::Operand';
sub new { my $package = shift; return bless { line => shift, members => [@_] }, $package; };
sub members { return @{$_[0]->{members}}; }

# Parentheses
package WWW::Shopify::Liquid::Token::Grouping::Parenthetical;
use base 'WWW::Shopify::Liquid::Token::Grouping';

# Like a grouping, but not really.
# Square brackets
package WWW::Shopify::Liquid::Token::Array;
use base 'WWW::Shopify::Liquid::Token::Operand';

sub new {
	my ($package, $line, @members) = @_;
	# Check to see whether or not the incoming member array is separated by commas.
	# Should never begin with a comma, should never end with a comma, should always be
	# 101010101 in terms of data and separators. If this is not the case, then we
	# Should throw a lexing exception.
	die new WWW::Shopify::Liquid::Exception::Lexer::InvalidSeparator($line) unless
		int(grep { ($_ % 2) == 0 && $members[$_]->isa('WWW::Shopify::Liquid::Token::Separator') } (0..$#members)) == 0 &&
		int(grep { ($_ % 2) == 1 && (!$members[$_]->isa('WWW::Shopify::Liquid::Token::Separator') || $members[$_]->{core} ne ",") } (0..$#members)) == 0;

	my $self = bless {
		line => $line,
		members => [grep { !$_->isa('WWW::Shopify::Liquid::Token::Separator') } @members]
	}, $package;
	return $self;
};

sub members { return @{$_[0]->{members}}; }
sub process {
	my ($self, $hash, $action, $pipeline) = @_;
	my @members = $self->members;
	$members[$_] = $members[$_]->$action($pipeline, $hash) for (grep { !$self->is_processed($members[$_]) } (0..$#members));
	if ($action eq "optimize") {
		$self->{members}->[$_] = $_ for (grep { $self->is_processed($members[$_]) } 0..$#members);
	}
	return [@members];
}

# Curly brackets
package WWW::Shopify::Liquid::Token::Hash;
use base 'WWW::Shopify::Liquid::Token::Operand';

sub members { return @{$_[0]->{members}}; }
sub new {
	my ($package, $line, @members) = @_;

	die new WWW::Shopify::Liquid::Exception::Lexer::InvalidSeparator($line) unless
		int(grep { ($_ % 2) == 0 && $members[$_]->isa('WWW::Shopify::Liquid::Token::Separator') } (0..$#members)) == 0 &&
		int(grep {
			($_ % 4) == 1 && (!$members[$_]->isa('WWW::Shopify::Liquid::Token::Separator') || $members[$_]->{core} ne ":") ||
			($_ % 4) == 3 && (!$members[$_]->isa('WWW::Shopify::Liquid::Token::Separator') || $members[$_]->{core} ne ",")
		} (0..$#members)) == 0 &&
		int(grep { !$_->isa('WWW::Shopify::Liquid::Token::Separator') } @members) % 2 == 0;

	return bless {
		line => $line,
		members => [grep { !$_->isa('WWW::Shopify::Liquid::Token::Separator') } @members]
	}, $package;
};

sub process {
	my ($self, $hash, $action, $pipeline) = @_;
	my @members = $self->members;
	$members[$_] = $members[$_]->$action($pipeline, $hash) for (grep { !$self->is_processed($members[$_]) } (0..$#members));
	if ($action eq "optimize") {
		$self->{members}->[$_] = $_ for (grep { $self->is_processed($members[$_]) } 0..$#members);
	}
	return { @members };
}


package WWW::Shopify::Liquid::Token::Text;
use base 'WWW::Shopify::Liquid::Token::Operand';
sub new {
	my $self = { line => $_[1], core => $_[2] };
	my $package = $_[0];
	$package = 'WWW::Shopify::Liquid::Token::Text::Whitespace' if !defined $_[2] || $_[2] =~ m/^\s*$/;
	return bless $self, $package;
};
sub process { my ($self, $hash) = @_; return $self->{core}; }

package WWW::Shopify::Liquid::Token::Text::Whitespace;
use base 'WWW::Shopify::Liquid::Token::Text';

package WWW::Shopify::Liquid::Token::Tag;
use base 'WWW::Shopify::Liquid::Token';
sub new { return bless { line => $_[1], tag => $_[2], arguments => $_[3], strip_left => $_[4], strip_right => $_[5] }, $_[0] };
sub tag { return $_[0]->{tag}; }
sub stringify { return $_[0]->tag; }

package WWW::Shopify::Liquid::Token::Output;
use base 'WWW::Shopify::Liquid::Token';
sub new { return bless { line => $_[1], core => $_[2], strip_left => $_[3], strip_right => $_[4] }, $_[0]; };

package WWW::Shopify::Liquid::Token::Separator;
use base 'WWW::Shopify::Liquid::Token';


package WWW::Shopify::Liquid::Lexer;
use base 'WWW::Shopify::Liquid::Pipeline';
use Scalar::Util qw(looks_like_number blessed);

sub new { return bless { operators => {}, lexing_halters => {}, transparent_filters => {}, unparse_spaces => 0, parse_escaped_characters => 1 }, $_[0]; }
sub operators { return $_[0]->{operators}; }
sub unparse_spaces { $_[0]->{unparse_spaces} = $_[1] if defined $_[1]; return $_[0]->{unparse_spaces}; }
sub parse_escaped_characters { $_[0]->{parse_escaped_characters} = $_[1] if defined $_[1]; return $_[0]->{parse_escaped_characters}; }
sub register_operator {	$_[0]->{operators}->{$_} = $_[1] for ($_[1]->symbol); }
sub register_tag {
	my ($self, $package) = @_;
	$self->{lexing_halters}->{$package->name} = $package if $package->is_enclosing && $package->inner_halt_lexing;
}
sub register_filter {
	my ($self, $package) = @_;
	$self->{transparent_filters}->{$package->name} = $package if ($package->transparent);
}
sub transparent_filters { return $_[0]->{transparent_filters}; }

sub parse_token {
	my ($self, $line, $token) = @_;

	# Strip token of whitespace.
	return undef unless defined $token;
	$token =~ s/^\s*(.*?)\s*$/$1/;
	return WWW::Shopify::Liquid::Token::Operator->new($line, $token) if $self->operators->{$token};
	return WWW::Shopify::Liquid::Token::String->new($line, do {
		my $string = $1;
		if ($self->parse_escaped_characters) {
			$string =~ s/\\r/\r/g;
			$string =~ s/\\n/\n/g;
			$string =~ s/\\t/\t/g;
		}
		$string;
	}) if $token =~ m/^'(.*)'$/s || $token =~ m/^"(.*)"$/s;
	return WWW::Shopify::Liquid::Token::Number->new($line, $1) if looks_like_number($token);
	return WWW::Shopify::Liquid::Token::NULL->new() if $token eq '' || $token eq 'null';
	return WWW::Shopify::Liquid::Token::Separator->new($line, $token) if ($token eq ":" || $token eq ",");
	return WWW::Shopify::Liquid::Token::Array->new($line) if ($token eq "[]");
	return WWW::Shopify::Liquid::Token::Hash->new($line) if ($token eq "{}");
	# We're a variable. Let's see what's going on. Split along non quoted . and [ ] fields.
	my ($squot, $dquot, $start, @parts) = (0,0,0);
	#  customer['test']['b']
	my $open_square_bracket = 0;
	my $open_curly_bracket = 0;
	while ($token =~ m/(\.|\[|\]|\{|\}|(?<!\\)\"|(?<!\\)\'|\b$)/g) {
		my $sym = $&;
		my $begin = $-[0];
		my $end = $+[0];
		if (!$squot && !$dquot) {
			$open_square_bracket-- if ($sym && $sym eq "]");
			$open_curly_bracket-- if ($sym && $sym eq "}");
			if (($sym eq "." || $sym eq "]" || $sym eq "[" || $sym eq "{" || $sym eq "}" || !$sym) && $open_square_bracket == 0 && $open_curly_bracket == 0) {
				my $contents = substr($token, $start, $begin - $start);

				if (defined $contents && $contents !~ m/^\s*$/) {
					my @variables = ();
					if (!$sym || $sym eq "." || $sym eq "[") {
						@variables = $self->transparent_filters->{$contents} ? WWW::Shopify::Liquid::Token::Variable::Processing->new($line, $self->transparent_filters->{$contents}) : WWW::Shopify::Liquid::Token::String->new($line, $contents);
					}
					elsif ($sym eq "]") {
						@variables = $self->parse_expression($line, $contents);
						return WWW::Shopify::Liquid::Token::Array->new($line, @variables) if (int(@parts) == 0);
					}
					elsif ($sym eq "}") {
						@variables = $self->parse_expression($line, $contents);
						return WWW::Shopify::Liquid::Token::Hash->new($line, @variables) if (int(@parts) == 0);
					}
					if (int(@variables) > 0) {
						if (int(@variables) == 1) {
							push(@parts, @variables);
						}
						else {
							push(@parts, WWW::Shopify::Liquid::Token::Grouping->new($line, @variables)) ;
						}
					}
				}
			}
			$start = $end if $sym ne '"' && $sym ne "'" && !$open_curly_bracket && !$open_square_bracket;
			$open_square_bracket++ if ($sym && $sym eq "[");
			$open_curly_bracket++ if ($sym && $sym eq "{");

		}
		$squot = !$squot if $token eq "'";
		$dquot = !$dquot if $token eq '"';
	}
	return WWW::Shopify::Liquid::Token::Variable->new($line, @parts);
}

use utf8;
# Returns a single token repsending the whole a expression.
sub parse_expression {
	my ($self, $line, $exp) = @_;
	return () if !defined $exp || $exp eq '';
	my @tokens = ();
	my ($start_paren, $start_space, $level, $squot, $dquot, $start_sq, $sq_level, $start_hsh, $hsh_level) = (undef, 0, 0, 0, 0, undef, 0, undef, 0);
	# We regex along parentheses, quotation marks (both kinds), whitespace, and non-word-operators.
	# We sort along length, so that we make sure to get all the largest operators first, so that way if a larger operator is made from a smaller one (=, ==)
	# There's no confusion, we always try to match the largest first.
	my $non_word_operators = join("|", map { quotemeta($_) } grep { $_ =~ m/^\W+$/; } sort { length($b) <=> length($a) } keys(%{$self->operators}));
	while ($exp =~ m/(?:\(|\)|\]|\[|\}|\{|(?<!\\)["”“]|(?<!\\)['‘’]|(\s+|$)|($non_word_operators|,|:))/sg) {
		my ($rs, $re, $rc, $whitespace, $nword_op) = ($-[0], $+[0], $&, $1, $2);
		# Specifically to allow variables to have a - in them, and be treated as a long literal, instead of a minus sign.
		# This is terrible behaviour, but mimics Shopify's lexer.
		# Only if of course the entire first half of this ISN'T a number, though. 'cause that'd be insane.
		next if $nword_op && $nword_op eq "-" && $rs > 0 && substr($exp, $rs-1, 1) ne " " && substr($exp, 0, $rs) !~ m/\b\d+$/;
		if (!$squot && !$dquot) {
			$start_paren = $re if $rc eq "(" && $level++ == 0;
			$start_sq = $re if $rc eq "[" && $sq_level++ == 0;
			$start_hsh = $re if $rc eq "{" && $hsh_level++ == 0;
			# Deal with parentheses; always the highest level of operation, except when inside a square bracket.
			if ($rc eq ")" && --$level == 0 && $sq_level == 0) {
				$start_space = $re;
				push(@tokens, WWW::Shopify::Liquid::Token::Grouping->new($line, $self->parse_expression($line, substr($exp, $start_paren, $rs - $start_paren))));
			}
			--$sq_level if $rc eq "]";
			--$hsh_level if $rc eq "}";
			if ($level == 0 && $sq_level == 0 && $hsh_level == 0) {
				# If we're only spaces, that means we're a new a token.
				if (defined $whitespace || $nword_op) {
					if (defined $start_space) {
						my $contents = substr($exp, $start_space, $rs - $start_space);
						push(@tokens, $self->parse_token($line, $contents)) if $contents !~ m/^\s*$/;
					}
					push(@tokens, $self->parse_token($line, $nword_op)) if $nword_op;
					$start_space = $re;
				}
			}
		}
		$squot = !$squot if ($rc eq "'" && !$dquot);
		$dquot = !$dquot if ($rc eq '"' && !$squot);
	}
	die WWW::Shopify::Liquid::Exception::Lexer::UnbalancedBrace->new($line) unless $level == 0;
	# Go through and combine any -1 from OP NUM to NUM.
	my @ids = grep {
		$tokens[$_]->isa('WWW::Shopify::Liquid::Token::Number') &&
		$tokens[$_-1]->isa('WWW::Shopify::Liquid::Token::Operator') && $tokens[$_-1]->{core} eq "-" &&
		($_ == 1 || $tokens[$_-2]->isa('WWW::Shopify::Liquid::Token::Separator') || $tokens[$_-2]->isa('WWW::Shopify::Liquid::Token::Operator'))
	} 1..$#tokens;
	for (@ids) { $tokens[$_]->{core} *= -1; $tokens[$_-1] = undef; }
	@tokens = grep { defined $_ } @tokens;
	# Go through and combine colon separated arguments into named arguments.
	@ids = grep {
		($_ == 2 || !$tokens[$_-3]->isa('WWW::Shopify::Liquid::Token::Operator') || $tokens[$_-3]->{core} ne "|") &&
		$tokens[$_-2]->isa('WWW::Shopify::Liquid::Token::Variable') && int(@{$tokens[$_-2]->{core}}) == 1 && $tokens[$_-2]->{core}->[0]->isa('WWW::Shopify::Liquid::Token::String') &&
		$tokens[$_-1]->isa('WWW::Shopify::Liquid::Token::Separator') && $tokens[$_-1]->{core} eq ":" &&
		($tokens[$_]->isa('WWW::Shopify::Liquid::Token::Grouping') || $tokens[$_]->isa('WWW::Shopify::Liquid::Token::Variable') || $tokens[$_]->isa('WWW::Shopify::Liquid::Token::Number') || $tokens[$_]->isa('WWW::Shopify::Liquid::Token::String'))
	} (2..$#tokens);
	for (@ids) { $tokens[$_] = WWW::Shopify::Liquid::Token::Variable::Named->new($line, $tokens[$_-2]->{core}->[0]->{core}, $tokens[$_]); $tokens[$_-2] = undef; $tokens[$_-1] = undef; }
	@tokens = grep { defined $_ } @tokens;
	return @tokens;
}

sub parse_text {
	my ($self, $text) = @_;
	return () unless defined $text;
	my @tokens = ();
	my $start = 0;
	my $line = 1;
	my $column = 0;
	my $lexing_halter = undef;

	my $start_line = 1;
	my $start_column = 0;
	my $line_position = 0;

	return (WWW::Shopify::Liquid::Token::Text->new(1, '')) if $text eq '';

	# No need to worry about quotations; liquid overrides everything.
	# We specifically look for tags within tags due to lexing halters abilities to be placed pretty much anywhere.
	# Oh god no, we have to rewrite this so nested output braces inside strings inside output braces are parsed properly.
	#while ($text =~ m/(?:(?:{%\s*(\w+)\s*(.*?)(?:\s*%}|\s*(?={%)))|(?:{{\s*(.*?)\s*}})|(\n)|$)/sg) {
	my $open_tag = undef;
	my $open_output_tag = undef;
	my $open_single_quote = undef;
	my $open_double_quote = undef;
	my $start_tag_strip = undef;
	my $end_tag_strip = undef;
	my @lines = split(/\n/, $text);
	while ($text =~ m/(?:(?:({%\-?)|(\-?%}))|(?:(\{\{\-?)|(\-?\}\}))|(\n)|(")|(')|$)/sg) {
		my ($start_tag, $end_tag, $start_output_tag, $end_output_tag, $newline, $double_quote, $single_quote) = ($1, $2, $3, $4, $5, $6, $7);
		next if ($double_quote || $single_quote) && !$open_tag && !$open_output_tag;
		my $column = $+[0] - $line_position;
		my $position = [$start_line, $start_column, $-[0], $self->file_context];
		my $end_position = [$start_line, $start_column, $+[0], $self->file_context];

		if ($start_output_tag || $start_tag) {
			$start_tag_strip = index(($start_output_tag || $start_tag), "-") != -1;
			$end_tag_strip = undef;
		} elsif ($end_output_tag || $end_tag) {
			$end_tag_strip = index(($end_output_tag || $end_tag), "-") != -1;
		}

		if (!$lexing_halter) {
			if ($double_quote || $single_quote) {
				if (!$open_double_quote && $single_quote) {
					$open_single_quote = $open_single_quote ? undef : $position;
				} elsif (!$open_single_quote && $double_quote) {
					$open_double_quote = $open_double_quote ? undef : $position;
				}
				next;
			}
			next if $open_single_quote || $open_double_quote;

			die new WWW::Shopify::Liquid::Exception::Lexer::UnbalancedTag([$line, ($-[0] - $line_position), $-[0]]) if ($end_tag && !defined $open_tag);
			# Unbalanced outputs tags shouldn't throw a lexer exception, due to CSS rules. We just treat this as part of text.
			# die new WWW::Shopify::Liquid::Exception::Lexer::UnbalancedOutputTag($position) if $end_output_tag && !defined $open_output_tag;
			next if $end_output_tag && !defined $open_output_tag;
		}
		if ($newline) {
			++$line;
			$line_position = $+[0];
			next;
		}
		if ($start_tag) {
			$open_tag = $position;
			next;
		}
		if ($start_output_tag) {
			$open_output_tag = $position;
			next;
		}

		my ($tag, $arguments, $output);
		my $start_element = undef;
		my $end_element = undef;
		if ($end_tag && $open_tag) {
			my $tag_line = substr($text, $open_tag->[2]+2, $-[0] - $open_tag->[2] - 2);
			$start_element = $open_tag;
			$end_element = $end_position;
			$open_tag = undef;
			die new WWW::Shopify::Liquid::Exception::Lexer::Tag($start_element, $tag_line) unless $tag_line =~ m/\s*(\w+)\s*(.*?)\s*$/s;
			($tag, $arguments) = ($1, $2);
		} elsif ($end_output_tag && $open_output_tag) {
			$start_element = $open_output_tag;
			$end_element = $end_position;
			$output = substr($text, $open_output_tag->[2]+2, $-[0] - $open_output_tag->[2] - 2);
			$output =~ s/(^\s*|\s*$)//g;
			$open_output_tag = undef;
		} else {
			$start_element = $position;
			$end_element = $end_position;
		}

		if ($tag && !$lexing_halter && exists $self->{lexing_halters}->{$tag}) {
			$lexing_halter = $tag;
			push(@tokens, WWW::Shopify::Liquid::Token::Text->new($position, substr($text, $start, $start_element->[2] - $start), $start_tag_strip, $end_tag_strip)) if ($start < $start_element->[2]);
			push(@tokens, WWW::Shopify::Liquid::Token::Tag->new($position, $tag, [$self->parse_expression($position, $arguments)], $start_tag_strip, $end_tag_strip));
			$start = $end_element->[2];
			$start_line = $line;
			$start_column = $start_element->[2] - $line_position;
			$position = [$start_line, $start_column, $start_element->[2], $self->file_context];
			$start_element = $position;
		}
		elsif ($tag && $lexing_halter && $tag eq "end" . $lexing_halter) {
			$lexing_halter = undef;
		}
		if (!$lexing_halter) {
			if ($start < $start_element->[2]) {
				push(@tokens, WWW::Shopify::Liquid::Token::Text->new([$start_line, $start_column, $start, $self->file_context], substr($text, $start, $start_element->[2] - $start), $start_tag_strip, $end_tag_strip));
				# Strip whitespace.
				if (int(@tokens) > 1 && $tokens[-1]->isa('WWW::Shopify::Liquid::Token::Text') && ($tokens[-2]->isa('WWW::Shopify::Liquid::Token::Tag') || $tokens[-2]->isa('WWW::Shopify::Liquid::Token::Output')) && $tokens[-2]->{strip_right}) {
					if ($tokens[-1]->isa('WWW::Shopify::Liquid::Token::Text::Whitespace')) {
						splice(@tokens, -1, 1);
					} else {
						$tokens[-1]->{core} =~ s/^\s+//ms;
					}
				}

				$start_line = $line;
				$start_column = $start_element->[2] - $line_position;
				$position = [$start_line, $start_column, $start_element->[2], $self->file_context];
				$start_element = $position;
			}
			push(@tokens, WWW::Shopify::Liquid::Token::Tag->new($start_element, $tag, [$self->parse_expression($position, $arguments)], $start_tag_strip, $end_tag_strip)) if $tag;
			push(@tokens, WWW::Shopify::Liquid::Token::Output->new($start_element, [$self->parse_expression($position, $output)], $start_tag_strip, $end_tag_strip)) if $output;
			$start = $end_element->[2];
			$start_line = $line;
			$start_column = $column;
		}
		# Strip whitespace.
		if ($tag && $start_tag_strip && int(@tokens) > 1 && $tokens[-2]->isa('WWW::Shopify::Liquid::Token::Text')) {
			if ($tokens[-2]->isa('WWW::Shopify::Liquid::Token::Text::Whitespace')) {
				splice(@tokens, -2, 1);
			} else {
				$tokens[-2]->{core} =~ s/\s+$//ms;
			}
		}

	}
	die new WWW::Shopify::Liquid::Exception::Lexer::UnbalancedSingleQuote($open_single_quote) if $open_single_quote;
	die new WWW::Shopify::Liquid::Exception::Lexer::UnbalancedDoubleQuote($open_double_quote) if $open_double_quote;
	die new WWW::Shopify::Liquid::Exception::Lexer::UnbalancedTag($open_tag) if $open_tag;
	die new WWW::Shopify::Liquid::Exception::Lexer::UnbalancedLexingHalt() if $lexing_halter;
	die new WWW::Shopify::Liquid::Exception::Lexer::UnbalancedOutputTag($open_output_tag) if $open_output_tag;
	return @tokens;
}

sub unparse_token {
	my ($self, $token) = @_;
	return "null" if !defined $token;
	return '' if $token eq '';
	return "'" . do { $token =~ s/'/\\'/g; $token } . "'" if !blessed($token) && !looks_like_number($token);
	return $token unless blessed($token);
	return "null" if $token->isa('WWW::Shopify::Liquid::Token::NULL');
	sub translate_variable {
		my ($self, $token) = @_;
		my $variable = join(".", map {
			!$self->is_processed($_) ? (
				$_->isa('WWW::Shopify::Liquid::Token::Variable') ? ('[' . translate_variable($self, $_) . ']') : $_->{core}
			) : $_
		} @{$token->{core}});
		$variable =~ s/\.\[/\[/g;
		return $variable;
	}

	return translate_variable($self, $token) if $token->isa('WWW::Shopify::Liquid::Token::Variable');
	return "(" . join("", map { $self->unparse_token($_); } @{$token->{members}}) . ")" if $token->isa('WWW::Shopify::Liquid::Token::Grouping');
	return "'" . $token->{core} . "'" if $token->isa('WWW::Shopify::Liquid::Token::String');
	return $token->{core};
}

sub unparse_expression {
	my ($self, @tokens) = @_;
	my $a = join("", map { ($_ eq ":" || $_ eq ",") ? $_ : " " . $_; } grep { defined $_ } map { $self->unparse_token($_) } @tokens);
	$a =~ s/^ //;
	return $a;
}

sub unparse_text_segment {
	my ($self, $token) = @_;
	my $space = $self->unparse_spaces ? " " : "";
	return $token if $self->is_processed($token);
	if ($token->isa('WWW::Shopify::Liquid::Token::Tag')) {
		return "{%" . $space . $token->{tag} . $space . "%}" if !$token->{arguments} || int(@{$token->{arguments}}) == 0;
		return "{%" . $space . $token->{tag} . " " . $self->unparse_expression(@{$token->{arguments}}) . $space  . "%}";
	}
	return "{{" . $space . $self->unparse_expression(@{$token->{core}}) . $space . "}}" if $token->isa('WWW::Shopify::Liquid::Token::Output');
	return $token->{core};
}

sub unparse_text {
	my ($self, @tokens) = @_;
	return join('', grep { defined $_ } map { $self->unparse_text_segment($_) } @tokens);
}

1;