Commit 02255eab authored by Roman Lacko's avatar Roman Lacko
Browse files

GitLab::API::Iterator added to handle paginated results

parent e1e3910f
......@@ -3,9 +3,11 @@ package GitLab::API;
use utf8;
use strict;
use warnings;
use vars qw($VERSION);
use Carp qw();
use Data::Dumper;
use GitLab::API::Iterator;
use HTTP::Headers;
use JSON;
use LWP::UserAgent;
......@@ -17,6 +19,8 @@ use URI::QueryParam;
use parent "Exporter";
our $VERSION = 8.12.1;
#===============================================================================
# Constructor
#===============================================================================
......@@ -185,45 +189,9 @@ sub clean_data {
#-- Request wrappers ---------------------------------------------------------
sub exec_request_get {
my ($self, $uri, $paginated) = @_;
my ($response, $data);
if (!$paginated) {
$response = $self->http->get($uri);
return ($response, $self->clean_data($response));
}
my $page = 0;
my $total;
$uri->query_param(per_page => 100);
do {
++$page;
$uri->query_param(page => $page);
$response = $self->http->get($uri);
return ($response, undef) if !$response->is_success;
if ($page == 1) {
$data = $self->clean_data($response);
croak "Request did not return an array"
if ref $data ne "ARRAY";
} else {
my $chunk = $self->clean_data($response);
croak "Request did not return an array"
if ref $chunk ne "ARRAY";
push @$data, @$chunk;
}
$total = $response->header("X-Total-Pages") unless defined $total;
} while defined $total && ($page < $total);
return ($response, $data);
my ($self, $uri) = @_;
my $response = $self->http->get($uri);
return ($response, $self->clean_data($response));
}
sub exec_request_post {
......@@ -270,8 +238,29 @@ sub exec_request {
my $response;
my $data = [];
if ($rtargs->{-iterator} && defined $rtargs->{-page}) {
carp "Cannot specify both -iterator and -page in method call";
}
$log->debug("$tmpl->{method} $uri");
if ($tmpl->{method} eq "GET") {
my $paginated = $tmpl->{paginated} && !defined $args->{page};
my $paginated = $tmpl->{paginated} && !defined $rtargs->{-page};
if (defined $rtargs->{-page}) {
$uri->query_param(page => $rtargs->{-page});
$uri->query_param(per_page => $rtargs->{-per_page} // 100);
}
if ($paginated) {
$log->debug("using iterator to obtain the result");
my $iterator = GitLab::API::Iterator->new($self, $uri,
per_page => $rtargs->{-per_page} // 100);
return $rtargs->{-iterator} ? $iterator : $iterator->all;
}
($response, $data) = $self->exec_request_get($uri, $paginated);
} elsif ($tmpl->{method} eq "POST") {
($response, $data) = $self->exec_request_post($uri, \@postdata);
......@@ -284,9 +273,9 @@ sub exec_request {
}
if ($log->is_trace) {
$log->trace("-- HTTP REQUEST --\n", $response->request->as_string);
$log->trace("-- HTTP RESPONSE --\n", $response->as_string);
$log->trace("--------------------");
$log->trace("-- HTTP REQUEST ", "-" x 25, "\n", $response->request->as_string);
$log->trace("-- HTTP RESPONSE ", "-" x 25, "\n", $response->as_string);
$log->trace("------------------", "-" x 25);
}
$log->debug("status: " . $response->status_line);
......
package GitLab::API::Iterator;
use utf8;
use strict;
use warnings;
use Carp qw();
use Log::Any qw($log);
our $VERSION = 8.12.1;
sub carp(@) { $log->warn(@_); Carp::carp @_; }
sub croak(@) { $log->fatal(@_); Carp::croak @_; }
#===============================================================================
# Constructor
#===============================================================================
sub new {
my ($class, $api, $url, %extra) = @_;
my $self = {
api => $api,
url => $url,
per_page => $extra{per_page} // 100,
};
bless $self, $class;
$self->reset;
$log->debug("created iterator for $url, per_page=$self->{per_page}");
return $self;
}
sub api { return shift->{api}; }
sub url { return shift->{url}; }
sub count { return shift->{count}; }
sub pages { return shift->{pages_read}; }
#===============================================================================
# Public methods
#===============================================================================
sub next {
my ($self) = @_;
return if ($self->{ix} + 1 == $self->count
&& ($self->{finished} || !$self->next_block));
++$self->{ix};
return $self->current;
}
sub reset {
my ($self) = @_;
$log->debug("resetting interator");
$self->{data} = [];
$self->{ix} = -1;
$self->{count} = 0;
$self->{pages_read} = 0;
$self->{pages_total} = undef;
$self->{finished} = 0;
$self->{good} = 1;
$self->{responses} = [];
}
sub rewind {
my ($self) = @_;
$log->debug("rewinding");
$self->{ix} = -1;
}
sub current {
my ($self) = @_;
return $self->{ix} == -1 || !$self->{good}
? undef
: $self->{data}->[$self->{ix}];
}
sub response {
my ($self, $index) = @_;
return !$self->{good}
? undef
: $self->{responses}->[$index];
}
sub all {
my ($self) = @_;
return undef if !$self->{good};
while ($self->next_block) {}
return !$self->{good} ? undef : $self->{data};
}
#===============================================================================
# Internals
#===============================================================================
sub next_block {
my ($self) = @_;
if (!$self->{good}) {
carp "Called next_block on an invalid iterator";
return 0;
}
$log->debug("requesting a new block, currently has $self->{pages_read} / " . ($self->{pages_total} // "unknown"));
if (defined $self->{pages_total} && $self->{pages_total} == $self->{pages_read}) {
$log->debug("all pages were read, nothing to do");
$self->{finished} = 1;
return 0;
}
my $xurl = $self->{url}->clone;
$xurl->query_param(page => $self->{pages_read} + 1);
$xurl->query_param(per_page => $self->{per_page});
$log->debug("GET " . $xurl);
my $response = $self->api->http->get($xurl);
my $data = $self->api->clean_data($response);
if ($log->is_trace) {
$log->trace("---- HTTP REQUEST ".("-" x 25)."\n".$response->request->as_string);
$log->trace("---- HTTP RESPONSE ".("-" x 25)."\n".$response->as_string);
$log->trace("--------------------".("-" x 25));
}
$log->debug("status: ", $response->status_line);
if (!$response->is_success) {
$self->{good} = 0;
croak "Cannot obtain next block: ", $response->status_line if $self->api->die_on_error;
return 0;
}
if (!defined $self->{pages_total}) {
$self->{pages_total} = $response->header("X-Total-Pages");
}
# if there was no X-Total-Pages, the response is not paginated, blame the user
if (!defined $self->{pages_total}) {
carp "Iterating over non-paginated request";
$self->{pages_total} = 1;
}
croak "Response did not contain an array"
unless ref $data eq "ARRAY";
push @{$self->{data}}, @$data;
push @{$self->{responses}}, $response;
$self->api->{last} = $response;
$self->{count} += scalar @$data;
++$self->{pages_read};
$self->{finished} = $self->{pages_read} == $self->{pages_total};
$log->debug("block read successful");
return 1;
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment