fix: big repos not crashing csv size

package-lock.json

@@ -11,6 +11,8 @@
         "@gitbeaker/rest": "^40.0.3",
         "@tanstack/react-query": "^5.51.1",
         "axios": "^1.7.2",
+        "axios-retry": "^4.5.0",
+        "es6-promise-pool": "^2.5.0",
         "next": "14.2.4",
         "react": "^18",
         "react-csv": "^2.2.2",
@@ -989,6 +991,17 @@
         "proxy-from-env": "^1.1.0"
       }
     },
+    "node_modules/axios-retry": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/axios-retry/-/axios-retry-4.5.0.tgz",
+      "integrity": "sha512-aR99oXhpEDGo0UuAlYcn2iGRds30k366Zfa05XWScR9QaQD4JYiP3/1Qt1u7YlefUOK+cn0CcwoL1oefavQUlQ==",
+      "dependencies": {
+        "is-retry-allowed": "^2.2.0"
+      },
+      "peerDependencies": {
+        "axios": "0.x || 1.x"
+      }
+    },
     "node_modules/axobject-query": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-3.1.1.tgz",
@@ -1627,6 +1640,14 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/es6-promise-pool": {
+      "version": "2.5.0",
+      "resolved": "https://registry.npmjs.org/es6-promise-pool/-/es6-promise-pool-2.5.0.tgz",
+      "integrity": "sha512-VHErXfzR/6r/+yyzPKeBvO0lgjfC5cbDCQWjWwMZWSb6YU39TGIl51OUmCfWCq4ylMdJSB8zkz2vIuIeIxXApA==",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/escape-string-regexp": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz",
@@ -2872,6 +2893,17 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/is-retry-allowed": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-2.2.0.tgz",
+      "integrity": "sha512-XVm7LOeLpTW4jV19QSH38vkswxoLud8sQ57YwJVTPWdiaI9I8keEhGFpBlslyVsgdQy4Opg8QOLb8YRgsyZiQg==",
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/is-set": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/is-set/-/is-set-2.0.3.tgz",

package.json

@@ -12,6 +12,8 @@
     "@gitbeaker/rest": "^40.0.3",
     "@tanstack/react-query": "^5.51.1",
     "axios": "^1.7.2",
+    "axios-retry": "^4.5.0",
+    "es6-promise-pool": "^2.5.0",
     "next": "14.2.4",
     "react": "^18",
     "react-csv": "^2.2.2",

src/app/hooks/useCommitSize.ts

 import { useQuery } from "@tanstack/react-query";
 import axios from "axios";
+import PromisePool from 'es6-promise-pool';
 
-export const useCommitSize = (projectId, commitIds, token, host) => {
+export const useCommitSize = (projectId, commits, token, host) => {
     const platform = host.includes('gitlab') ? 'gitlab' : 'github';
 
-
     const { data, isLoading } = useQuery({
-        queryKey: ['commitDataSize', platform, projectId, commitIds],
+        queryKey: ['commitDataSize', platform, projectId, commits],
         queryFn: async () => {
-            const promises = commitIds.map(commitId => {
+            const BATCH_SIZE = 30;
+
+            const fetchCommitData = async (commit) => {
                 const url = platform === 'gitlab'
-                    ? `${host}/api/v4/projects/${projectId}/repository/commits/${commitId.id}`
-                    : `https://api.github.com/repos/${projectId}/commits/${commitId.sha}`;
+                    ? `${host}/api/v4/projects/${projectId}/repository/commits/${commit.id}`
+                    : `https://api.github.com/repos/${projectId}/commits/${commit.sha}`;
 
                 const headers = platform === 'gitlab'
                     ? { 'PRIVATE-TOKEN': token }
                     : { 'Authorization': `Bearer ${token}` };
 
-                return axios.get(url, { headers }).then(response => {
-                    const { stats, commit, author } = response.data;
-                    let commitSize = 'small';
-                    const totalChanges = stats.additions + stats.deletions;
-
-                    if (totalChanges > 50) commitSize = 'medium';
-                    if (totalChanges > 200) commitSize = 'big';
-
-                    return {
-                        project_id: projectId,
-                        author_name: platform === 'gitlab'
-                            ? response.data.author_name
-                            : author?.login || commit.author.name,
-                        created_at: platform === 'gitlab'
-                            ? response.data.created_at
-                            : commit.author.date,
-                        size: commitSize
-                    };
-                });
-            });
-
-            return Promise.all(promises);
+                const response = await axios.get(url, { headers });
+                const { stats, commit: commitData, author } = response.data;
+                let commitSize = 'small';
+                const totalChanges = stats.additions + stats.deletions;
+
+                if (totalChanges > 50) commitSize = 'medium';
+                if (totalChanges > 200) commitSize = 'big';
+
+                return {
+                    project_id: projectId,
+                    author_name: platform === 'gitlab'
+                        ? response.data.author_name
+                        : author?.login || commitData.author.name,
+                    created_at: platform === 'gitlab'
+                        ? response.data.created_at
+                        : commitData.author.date,
+                    size: commitSize
+                };
+            };
+
+            const results = [];
+            const pool = new PromisePool(() => {
+                const commit = commits.shift();
+                return commit ? fetchCommitData(commit).then(result => {
+                    results.push(result);
+                    return result;
+                }) : null;
+            }, BATCH_SIZE);
+
+            await pool.start();
+            console.log('results', results);
+            return Promise.all(results);
         }
     });