Merge pull request #450 from scholarly-python-package/develop

Reorganize tests and fix failing ones

Author: arunkannawadi

.github/workflows/lint.yaml

@@ -0,0 +1,26 @@
+name: lint
+
+on:
+  workflow_call:
+  push:
+    branches: [main, develop]
+  pull_request:
+    branches: [main, develop]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v3
+        with:
+          python-version: 3.8
+      - name: Lint with flake8
+        run: |
+          pip3 install -r requirements-dev.txt
+          # Stop the build if there are Python syntax errors, undefined names or unused imports etc.
+          flake8
+          # Do not stop the build, but indicate room for improvement.
+          # exit-zero treats all errors as warnings.
+          flake8 . --exit-zero --ignore= --select=

.github/workflows/proxytests.yml

@@ -0,0 +1,67 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Python package (with methods that need proxy)
+
+on:
+  workflow_run:
+    workflows: ["lint"]
+    branches: [main]
+    types:
+      - completed
+  pull_request:
+    branches: [main]
+
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 180
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: [3.8]
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python3 -m pip install --upgrade pip
+          pip3 install -r requirements-dev.txt
+          pip3 install -r requirements.txt
+      #- name: Typilus, Suggest Python Type Annotations
+      #  uses: typilus/typilus-action@v0.9
+      - name: Run unittests
+        id: unittests
+        if: ${{ github.event.pull_request.draft == false }}
+        continue-on-error: true
+        env:
+          CONNECTION_METHOD: ${{ secrets.CONNECTION_METHOD }}
+          PASSWORD: ${{ secrets.PASSWORD }}
+          PORT: ${{ secrets.PORT }}
+          USERNAME: ${{ secrets.USERNAME }}
+          SCRAPER_API_KEY: ${{ secrets.SCRAPER_API_KEY }}
+        run: |
+          coverage run -m unittest -v test_module.TestScholarlyWithProxy
+      - name: Generate coverage report
+        if:
+          "matrix.os == 'macos-latest'"
+        run: |
+          curl --data-binary @.github/.codecov.yml https://codecov.io/validate | head -n 1
+          coverage xml
+      - name: Upload code coverage
+        uses: codecov/codecov-action@v2
+        with:
+          directory: ./
+          fail_ci_if_error: false
+          files: ./coverage.xml
+          flags: unittests
+          name: codecov-umbrella
+          verbose: true
+      - name: Check if unittests failed
+        if: "steps.unittests.outcome == 'failure'"
+        run: exit 1

.github/workflows/pythonpackage.yml

@@ -10,10 +10,18 @@ on:
     branches: [main, develop]
   pull_request:
     branches: [main, develop]
+  workflow_run:
+    workflows: ["lint"]
+#    branches: [main]
+    types:
+      - completed
 
 jobs:
+  lint:
+    uses: ./.github/workflows/lint.yaml
   build:
     runs-on: ${{ matrix.os }}
+    needs: [lint]
     timeout-minutes: 180
     strategy:
       fail-fast: false
@@ -27,29 +35,16 @@ jobs:
         uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Lint with flake8
-        run: |
-          pip3 install -r requirements-dev.txt
-          # Stop the build if there are Python syntax errors, undefined names or unused imports etc.
-          flake8
-          # Do not stop the build, but indicate room for improvement.
-          # exit-zero treats all errors as warnings.
-          flake8 . --exit-zero --ignore= --select=
       - name: Install dependencies
         run: |
           python3 -m pip install --upgrade pip
           pip3 install -r requirements.txt
+          pip3 install -r requirements-dev.txt
       #- name: Typilus, Suggest Python Type Annotations
       #  uses: typilus/typilus-action@v0.9
       - name: Run unittests
         id: unittests
         continue-on-error: true
-        env:
-          CONNECTION_METHOD: ${{ secrets.CONNECTION_METHOD }}
-          PASSWORD: ${{ secrets.PASSWORD }}
-          PORT: ${{ secrets.PORT }}
-          USERNAME: ${{ secrets.USERNAME }}
-          SCRAPER_API_KEY: ${{ secrets.SCRAPER_API_KEY }}
         run: |
           coverage run -m unittest -v test_module.TestScholarly
       - name: Generate coverage report (macOS only)

scholarly/publication_parser.py

@@ -344,6 +344,7 @@ def fill(self, publication: Publication)->Publication:
                     for entry in val.find_all('a'):
                         if entry.text.lower() == 'related articles':
                             publication['url_related_articles'] = entry.get('href')[26:]
+                            break
             # number of citation per year
             years = [int(y.text) for y in soup.find_all(class_='gsc_oci_g_t')]
             cites = [int(c.text) for c in soup.find_all(class_='gsc_oci_g_al')]