Merge pull request #12 from soda480/0.0.9

Fix paging

Author: soda480

README.md

@@ -66,8 +66,9 @@ for repo in client.get('/user/repos', _get='all', _attributes=['full_name']):
 
 `GET page` directive - Yield a page from endpoint
 ```python
-for repo in client.get('/user/repos', _get='page'):
-    print(repo['full_name'])
+for page in client.get('/user/repos', _get='page'):
+    for repo in page:
+        print(repo['full_name'])
 ```
 
 ### Projects using `github3api` ###

build.py

@@ -30,7 +30,7 @@
 authors = [Author('Emilio Reyes', 'emilio.reyes@intel.com')]
 summary = 'An advanced REST client for the GitHub API'
 url = 'https://github.com/soda480/github3api'
-version = '0.0.8'
+version = '0.0.9'
 default_task = [
     'clean',
     'analyze',

src/main/python/github3api/githubapi.py

@@ -55,37 +55,33 @@ def get_headers(self, **kwargs):
         headers['Accept'] = f'application/vnd.github.{self.version}+json'
         return headers
 
-    def _get_next_endpoint(self, link_header):
-        """ return next endpoint from link header
+    def _get_next_endpoint(self, url):
+        """ return next endpoint
         """
-        if not link_header:
+        if not url:
             logger.debug('link header is empty')
             return
-        regex = fr".*<https://{self.hostname}(?P<endpoint>/.*?)>; rel=\"next\".*"
-        match = re.match(regex, link_header)
-        if match:
-            endpoint = match.group('endpoint')
-            logger.debug(f'found next endpoint in link header: {endpoint}')
-            return endpoint
-        logger.debug('next endpoints not found in link header')
+        endpoint = url.replace(f'https://{self.hostname}', '')
+        logger.debug(f'next endpoint is: {endpoint}')
+        return endpoint
 
     def _get_all(self, endpoint, **kwargs):
         """ return all pages from endpoint
         """
         logger.debug(f'get items from: {endpoint}')
         items = []
         while True:
-            link_header = None
+            url = None
             response = super(GitHubAPI, self).get(endpoint, raw_response=True, **kwargs)
             if response:
                 data = response.json()
                 if isinstance(data, list):
                     items.extend(response.json())
                 else:
                     items.append(data)
-                link_header = response.headers.get('Link')
+                url = response.links.get('next', {}).get('url')
 
-            endpoint = self._get_next_endpoint(link_header)
+            endpoint = self._get_next_endpoint(url)
             if not endpoint:
                 logger.debug('no more pages to retrieve')
                 break
@@ -97,9 +93,8 @@ def _get_page(self, endpoint, **kwargs):
         """
         while True:
             response = super(GitHubAPI, self).get(endpoint, raw_response=True, **kwargs)
-            for page in response.json():
-                yield page
-            endpoint = self._get_next_endpoint(response.headers.get('Link'))
+            yield response.json()
+            endpoint = self._get_next_endpoint(response.links.get('next', {}).get('url'))
             if not endpoint:
                 logger.debug('no more pages')
                 break

src/unittest/python/test_githubapi.py

@@ -176,17 +176,11 @@ def test__get_next_endpoint_Should_ReturnNone_When_NoLinkHeader(self, *patches):
 
     def test__get_next_endpoint_Should_ReturnExpected_When_CalledWithNextEndpoint(self, *patches):
         client = GitHubAPI(bearer_token='bearer-token')
-        link_header = '<https://api.github.com/organizations/27781926/repos?page=2>; rel="prev", <https://api.github.com/organizations/27781926/repos?page=4>; rel="next", <https://api.github.com/organizations/27781926/repos?page=4>; rel="last", <https://api.github.com/organizations/27781926/repos?page=1>; rel="first"'
+        link_header = 'https://api.github.com/organizations/27781926/repos?page=4'
         result = client._get_next_endpoint(link_header)
         expected_result = '/organizations/27781926/repos?page=4'
         self.assertEqual(result, expected_result)
 
-    def test__get_next_endpoint_Should_ReturnNone_When_NoNextEndpoint(self, *patches):
-        client = GitHubAPI(bearer_token='bearer-token')
-        link_header = '<https://api.github.com/organizations/27781926/repos?page=3>; rel="prev", <https://api.github.com/organizations/27781926/repos?page=1>; rel="first"'
-        result = client._get_next_endpoint(link_header)
-        self.assertIsNone(result)
-
     @patch('github3api.GitHubAPI._get_next_endpoint')
     @patch('github3api.githubapi.RESTclient.get')
     def test__get_all_Should_ReturnExpected_When_GetReturnsList(self, get_patch, get_next_endpoint_patch, *patches):
@@ -264,10 +258,8 @@ def test__get_page_Should_ReturnExpected_When_Called(self, get_patch, get_next_e
         ]
         client = GitHubAPI(bearer_token='bearer-token')
         result = client._get_page('endpoint')
-        self.assertEqual(next(result), 'page1')
-        self.assertEqual(next(result), 'page2')
-        self.assertEqual(next(result), 'page3')
-        self.assertEqual(next(result), 'page4')
+        self.assertEqual(next(result), ['page1', 'page2'])
+        self.assertEqual(next(result), ['page3', 'page4'])
         with self.assertRaises(StopIteration):
             next(result)
 
@@ -287,8 +279,7 @@ def test__get_page_Should_ReturnExpected_When_NoEndpoint(self, get_patch, get_ne
         ]
         client = GitHubAPI(bearer_token='bearer-token')
         result = client._get_page('endpoint')
-        self.assertEqual(next(result), 'page1')
-        self.assertEqual(next(result), 'page2')
+        self.assertEqual(next(result), ['page1', 'page2'])
         with self.assertRaises(StopIteration):
             next(result)