Commit 02255eab authored by Roman Lacko's avatar Roman Lacko
Browse files

GitLab::API::Iterator added to handle paginated results

parent e1e3910f

GitLab/API.pm

+32 −43
Original line number Diff line number Diff line
@@ -3,9 +3,11 @@ package GitLab::API; 
use utf8;

use strict;

use warnings;

use vars qw($VERSION);



use Carp            qw();

use Data::Dumper;

use GitLab::API::Iterator;

use HTTP::Headers;

use JSON;

use LWP::UserAgent;
@@ -17,6 +19,8 @@ use URI::QueryParam; 


use parent "Exporter";



our $VERSION = 8.12.1;



#===============================================================================

#   Constructor

#===============================================================================
@@ -185,47 +189,11 @@ sub clean_data { 
#--  Request wrappers  ---------------------------------------------------------



sub exec_request_get {

    my ($self, $uri, $paginated) = @_;



    my ($response, $data);



    if (!$paginated) {

        $response = $self->http->get($uri);

    my ($self, $uri) = @_;

    my $response = $self->http->get($uri);

    return ($response, $self->clean_data($response));

}



    my $page = 0;

    my $total;



    $uri->query_param(per_page => 100);



    do {

        ++$page;

        $uri->query_param(page => $page);



        $response = $self->http->get($uri);

        return ($response, undef) if !$response->is_success;



        if ($page == 1) {

            $data = $self->clean_data($response);



            croak "Request did not return an array"

                if ref $data ne "ARRAY";

        } else {

            my $chunk = $self->clean_data($response);



            croak "Request did not return an array"

                if ref $chunk ne "ARRAY";



            push @$data, @$chunk;

        }



        $total = $response->header("X-Total-Pages") unless defined $total;

    } while defined $total && ($page < $total);



    return ($response, $data);

}



sub exec_request_post {

    my ($self, $uri, $data) = @_;

    my $response = $self->http->post($uri, $data);
@@ -270,8 +238,29 @@ sub exec_request { 
    my $response;

    my $data        = [];



    if ($rtargs->{-iterator} && defined $rtargs->{-page}) {

        carp "Cannot specify both -iterator and -page in method call";

    }



    $log->debug("$tmpl->{method} $uri");



    if ($tmpl->{method} eq "GET") {

        my $paginated = $tmpl->{paginated} && !defined $args->{page};

        my $paginated = $tmpl->{paginated} && !defined $rtargs->{-page};



        if (defined $rtargs->{-page}) {

            $uri->query_param(page      => $rtargs->{-page});

            $uri->query_param(per_page  => $rtargs->{-per_page} // 100);

        }



        if ($paginated) {

            $log->debug("using iterator to obtain the result");



            my $iterator = GitLab::API::Iterator->new($self, $uri,

                per_page => $rtargs->{-per_page} // 100);



            return $rtargs->{-iterator} ? $iterator : $iterator->all;

        }



        ($response, $data) = $self->exec_request_get($uri, $paginated);

    } elsif ($tmpl->{method} eq "POST") {

        ($response, $data) = $self->exec_request_post($uri, \@postdata);
@@ -284,9 +273,9 @@ sub exec_request { 
    }



    if ($log->is_trace) {

        $log->trace("--  HTTP REQUEST  --\n", $response->request->as_string);

        $log->trace("--  HTTP RESPONSE --\n", $response->as_string);

        $log->trace("--------------------");

        $log->trace("--  HTTP REQUEST  ", "-" x 25, "\n", $response->request->as_string);

        $log->trace("--  HTTP RESPONSE ", "-" x 25, "\n", $response->as_string);

        $log->trace("------------------", "-" x 25);

    }



    $log->debug("status: " . $response->status_line);

GitLab/API/Iterator.pm

0 → 100644
+164 −0
Original line number Diff line number Diff line 
package GitLab::API::Iterator;



use utf8;

use strict;

use warnings;



use Carp        qw();

use Log::Any    qw($log);



our $VERSION = 8.12.1;



sub carp(@)  { $log->warn(@_);  Carp::carp  @_; }

sub croak(@) { $log->fatal(@_); Carp::croak @_; }



#===============================================================================

#   Constructor

#===============================================================================



sub new {

    my ($class, $api, $url, %extra) = @_;



    my $self = {

        api         => $api,

        url         => $url,

        per_page    => $extra{per_page} // 100,

    };



    bless $self, $class;

    $self->reset;



    $log->debug("created iterator for $url, per_page=$self->{per_page}");

    return $self;

}



sub api     { return shift->{api};   }

sub url     { return shift->{url};   }

sub count   { return shift->{count}; }

sub pages   { return shift->{pages_read}; }



#===============================================================================

#   Public methods

#===============================================================================



sub next {

    my ($self) = @_;



    return if ($self->{ix} + 1 == $self->count

        && ($self->{finished} || !$self->next_block));



    ++$self->{ix};

    return $self->current;

}



sub reset {

    my ($self) = @_;



    $log->debug("resetting interator");

    $self->{data}           = [];

    $self->{ix}             = -1;

    $self->{count}          =  0;

    $self->{pages_read}     =  0;

    $self->{pages_total}    =  undef;

    $self->{finished}       =  0;

    $self->{good}           =  1;

    $self->{responses}      = [];

}



sub rewind {

    my ($self) = @_;



    $log->debug("rewinding");

    $self->{ix} = -1;

}



sub current {

    my ($self) = @_;

    return $self->{ix} == -1 || !$self->{good}

        ? undef

        : $self->{data}->[$self->{ix}];

}



sub response {

    my ($self, $index) = @_;

    return !$self->{good}

        ? undef

        : $self->{responses}->[$index];

}



sub all {

    my ($self) = @_;

    return undef if !$self->{good};



    while ($self->next_block) {}



    return !$self->{good} ? undef : $self->{data};

}



#===============================================================================

#   Internals

#===============================================================================



sub next_block {

    my ($self) = @_;



    if (!$self->{good}) {

        carp "Called next_block on an invalid iterator";

        return 0;

    }



    $log->debug("requesting a new block, currently has $self->{pages_read} / " . ($self->{pages_total} // "unknown"));



    if (defined $self->{pages_total} && $self->{pages_total} == $self->{pages_read}) {

        $log->debug("all pages were read, nothing to do");

        $self->{finished} = 1;

        return 0;

    }



    my $xurl = $self->{url}->clone;

    $xurl->query_param(page     => $self->{pages_read} + 1);

    $xurl->query_param(per_page => $self->{per_page});



    $log->debug("GET " . $xurl);

    my $response = $self->api->http->get($xurl);

    my $data     = $self->api->clean_data($response);



    if ($log->is_trace) {

        $log->trace("----  HTTP REQUEST  ".("-" x 25)."\n".$response->request->as_string);

        $log->trace("----  HTTP RESPONSE ".("-" x 25)."\n".$response->as_string);

        $log->trace("--------------------".("-" x 25));

    }



    $log->debug("status: ", $response->status_line);



    if (!$response->is_success) {

        $self->{good} = 0;



        croak "Cannot obtain next block: ", $response->status_line if $self->api->die_on_error;

        return 0;

    }



    if (!defined $self->{pages_total}) {

        $self->{pages_total} = $response->header("X-Total-Pages");

    }



    # if there was no X-Total-Pages, the response is not paginated, blame the user

    if (!defined $self->{pages_total}) {

        carp "Iterating over non-paginated request";

        $self->{pages_total} = 1;

    }



    croak "Response did not contain an array"

        unless ref $data eq "ARRAY";



    push @{$self->{data}},      @$data;

    push @{$self->{responses}}, $response;



    $self->api->{last} = $response;

    $self->{count} += scalar @$data;

    ++$self->{pages_read};



    $self->{finished} = $self->{pages_read} == $self->{pages_total};

    $log->debug("block read successful");

    return 1;

}