.

pkin/TODO

@@ -0,0 +1,2 @@
+fetch caching
+make pkin_extractjobs nproc(1) by default

pkin/asset/Makefile

@@ -0,0 +1,2 @@
+example.binpkg.tar:
+	cd example.binpkg; tar vcf ../$@ *

pkin/asset/example.a

@@ -1,2 +1,2 @@
-lr	1.0	foo	https://lab0x13.site/index.html
+lr	1.0	foo	http://localhost:8081/example.binpkg.tar
 asd	1.0	asd	http://kentoj/index.html

pkin/meson.build

@@ -11,14 +11,30 @@ deps = [
   cc.find_library('libowfat'),
 ]
 
+add_project_arguments(
+  '-DPATH_BIN_INTERNAL="'+get_option('path_bin_internal')+'"', 
+  language: 'c',
+)
+add_project_arguments(
+  '-DPATH_CACHE="'+get_option('path_cache')+'"', 
+  language: 'c',
+)
+
 executable('pkin', files(
-  './pkin.c',
-  './pkin-search.c',
-  './pkin-fetch.c',
-  './pkin/pkin_repo_iter_init.c',
-  './pkin/pkin_repo_iter_read.c',
-  './pkin/pkin_pkg_fetch_prepare.c',
-  './pkin/pkin_pkg_fetch_add.c',
-  './pkin/pkin_pkg_fetch_start.c',
-  './pkin_internal/pkin__spawn.c',
+  './src/bin/pkin.c',
+  './src/bin/pkin-search.c',
+  './src/bin/pkin-fetch.c',
+  './src/pkin/pkin_repo_iter_init.c',
+  './src/pkin/pkin_repo_iter_read.c',
+  './src/pkin/pkin_queue_prepare.c',
+  './src/pkin/pkin_queue_extract_add.c',
+  './src/pkin/pkin_queue_extract_start.c',
+  './src/pkin/pkin_queue_fetch_add.c',
+  './src/pkin/pkin_queue_fetch_start.c',
+  './src/pkin_internal/pkin__spawn.c',
+  './src/pkin_internal/pkin__mkpath.c',
 ), dependencies:deps)
+
+executable('extraction-worker', files(
+  './src/bin-internal/extraction-worker.c',
+), dependencies:deps + dependency('libarchive'))

pkin/meson.options

@@ -0,0 +1,13 @@
+option(
+  'path_bin_internal',
+  type : 'string',
+  value : '/usr/libexec/pkin',
+  description : 'path to internal binaries',
+)
+
+option(
+  'path_cache',
+  type : 'string',
+  value : '/var/cache/pkin',
+  description : 'path to cache',
+)

pkin/pkin/pkin_pkg_fetch_add.c

@@ -1,10 +0,0 @@
-#include "../pkin.h"
-
-void pkin_fetch_add(pkin_fetch_queue *queue, char *url, char *output)
-{
-  buffer_puts(&queue->writeb, url);
-  buffer_puts(&queue->writeb, "\n out=");
-  buffer_puts(&queue->writeb, output);
-  buffer_puts(&queue->writeb, "\n user-agent=pkin <https://git.lab0x13.site/kento2/pkin>");
-  buffer_puts(&queue->writeb, "\n");
-}

pkin/pkin/pkin_pkg_fetch_start.c

@@ -1,34 +0,0 @@
-#include <libowfat/errmsg.h>
-#include <libowfat/open.h>
-#include <sys/wait.h>
-#include <unistd.h>
-#include "../pkin.h"
-#include "../pkin_internal.h"
-
-int pkin_fetch_start(pkin_fetch_queue *queue)
-{
-  char **fetchcmd;
-  int nullfd;
-  pid_t pid;
-  if (buffer_flush(&queue->writeb) < 0) {
-    carpsys("write to pipe");
-    return 0;
-  }
-  close(queue->pipefd[1]);
-
-  fetchcmd = (char *[]){"aria2c", "-i", "-", NULL};
-  nullfd = open_write("/dev/null");
-  pid = pkin__spawn(fetchcmd, queue->pipefd[0], nullfd < 0 ? 1 : nullfd, 2);
-  if (pid < 0) {
-    close(nullfd);
-    carp("failed spawning aria2c to fetch files");
-    return 0;
-  }
-
-  if (waitpid(pid, NULL, 0) < 0) {
-    close(nullfd);
-    carp("waitpid");
-    return 0;
-  }
-  return 1;
-}

pkin/src/bin-internal/extraction-worker.c

@@ -0,0 +1,125 @@
+/*
+ * see also: https://github.com/libarchive/libarchive/blob/master/examples/untar.c
+ */
+
+#include <libowfat/stralloc.h>
+#include <libowfat/buffer.h>
+#include <libowfat/errmsg.h>
+#include <libowfat/str.h>
+#include <archive.h>
+#include <archive_entry.h>
+#include <alloca.h>
+
+#define errstr(a) archive_error_string(a)
+
+static int copy_data(struct archive *reader, struct archive *writer)
+{
+  int r;
+  const void *buf;
+  size_t size;
+#if ARCHIVE_VERSION_NUMBER >= 3000000
+	int64_t offset;
+#else
+	off_t offset;
+#endif
+  for (;;) {
+    r = archive_read_data_block(reader,&buf,&size,&offset);
+    if (r == ARCHIVE_EOF) return 1;
+    if (r != 0) {
+      carpsys("archive_read_data_block: ", errstr(reader));
+      return 0;
+    }
+    r = archive_write_data_block(writer,buf,size,offset);
+    if (r != 0) {
+      carpsys("archive_write_data_block: ", errstr(reader));
+      return 0;
+    }
+  }
+  return 1;
+}
+
+int extract(char *in, const char *out)
+{
+  struct archive *reader = NULL, *writer = NULL;
+  struct archive_entry *ent;
+  int ret = 0;
+  int r;
+  size_t l;
+  const char *oldpath;
+  char *newpath;
+
+  reader = archive_read_new();
+  archive_read_support_filter_all(reader);
+  archive_read_support_format_all(reader);
+  writer = archive_write_disk_new();
+  archive_write_disk_set_options(writer, 
+      ARCHIVE_EXTRACT_PERM|ARCHIVE_EXTRACT_ACL|ARCHIVE_EXTRACT_FFLAGS);
+
+  if (archive_read_open_filename(reader, in, 8192) !=  0) {
+    carp("open archive ", in, " for reading: ", errstr(reader));
+    goto done;
+  }
+
+  for (;;) {
+    r = archive_read_next_header(reader, &ent);
+    if (r == ARCHIVE_EOF) break;
+    if (r != 0) {
+      carp("failed to read archive header: ", errstr(reader));
+      goto done;
+    }
+
+    oldpath = archive_entry_pathname(ent);
+    newpath = alloca(str_len(oldpath) + str_len(out) + 1);
+    l = str_copy(newpath, out);
+    l += str_copy(newpath + l, "/");
+    str_copy(newpath + l, oldpath);
+    msg(oldpath, " -> ", newpath);
+    archive_entry_set_pathname(ent, newpath);
+
+    if (archive_write_header(writer, ent) != 0) {
+      carp("failed to write header ", errstr(writer));
+      continue;
+    }
+    if (!copy_data(reader, writer)) {
+      carp("failed to copy archive data");
+      goto done;
+    }
+    if (archive_write_finish_entry(writer) != 0) {
+      carp("archive_write_finish_entry: ", errstr(writer));
+      goto done;
+    }
+  }
+
+  ret = 1;
+done:
+  if (reader) {
+    archive_read_close(reader);
+    archive_read_free(reader);
+  }
+  if (writer) {
+    archive_write_close(writer);
+    archive_write_free(writer);
+  }
+  return ret;
+}
+
+int main()
+{
+  stralloc line;
+  int r;
+  size_t l;
+
+  for (;;) {
+    r  =  buffer_getnewline_sa(buffer_0,  &line);
+    if (r == 0) break;
+    if (r < 0) diesys(111, "failed to read stdin");
+    stralloc_chomp(&line);
+    stralloc_0(&line);
+    /* separate by space, first part is archive, rest is output dir */
+    l = str_chr(line.s, ' ');
+    if (l == str_len(line.s))
+      die(100, "malformed entry in stdin: missing separator(space)");
+    line.s[l] = 0;
+    extract(line.s, line.s + l + 1);
+  } 
+}

pkin/src/bin-internal/meson.build

@@ -0,0 +1,17 @@
+cc = meson.get_compiler('c')
+deps = [
+  cc.find_library('libowfat'),
+  dependency("libarchive"),
+]
+
+executable('pkin', files(
+  './pkin.c',
+  './pkin-search.c',
+  './pkin-fetch.c',
+  './pkin/pkin_repo_iter_init.c',
+  './pkin/pkin_repo_iter_read.c',
+  './pkin/pkin_pkg_fetch_prepare.c',
+  './pkin/pkin_pkg_fetch_add.c',
+  './pkin/pkin_pkg_fetch_start.c',
+  './pkin_internal/pkin__spawn.c',
+), dependencies:deps)

pkin/src/bin/meson.build

@@ -0,0 +1,16 @@
+cc = meson.get_compiler('c')
+deps = [
+  cc.find_library('libowfat'),
+]
+
+executable('pkin', files(
+  './pkin.c',
+  './pkin-search.c',
+  './pkin-fetch.c',
+  '../pkin_repo_iter_init.c',
+  '../pkin/pkin_repo_iter_read.c',
+  '../pkin/pkin_pkg_fetch_prepare.c',
+  '../pkin/pkin_pkg_fetch_add.c',
+  '../pkin/pkin_pkg_fetch_start.c',
+  '../pkin_internal/pkin__spawn.c',
+), dependencies:deps)

pkin/src/bin/pkin-fetch.c

@@ -22,18 +22,31 @@ static void run(int argc, char **argv)
 {
   int i;
   pkin_pkgmeta pkgmeta;
-  pkin_fetch_queue queue;
-  if (!pkin_fetch_prepare(&queue))
-    die(111, "failed to create package fetch queue");
+  pkin_queue fetchq, extractq;
+  stralloc archivename;
+  if (!pkin_queue_prepare(&fetchq)
+   || !pkin_queue_prepare(&extractq)
+  ) die(111, "failed to create queue");
 
   for (i = 0; i < argc; i++) {
     if (!find_package(argv[i], &pkgmeta))
       die(100, "package '", argv[i], "' not found ");
-    pkin_fetch_add(&queue, pkgmeta.url, pkgmeta.name);
+    stralloc_copys(&archivename, pkgmeta.name);
+    stralloc_cats(&archivename, ".archive");
+    stralloc_0(&archivename);
+    pkin_queue_fetch_add(&fetchq, pkgmeta.url, archivename.s);
+    pkin_queue_extract_add(&extractq, archivename.s, pkgmeta.name);
+    if (pkin_verbose)
+      msg("queueing fetch/extract: ", pkgmeta.url);
   }
+  stralloc_free(&archivename);
 
-  if (!pkin_fetch_start(&queue))
+  msg("fetching binary packages");
+  if (!pkin_queue_fetch_start(&fetchq))
     die(111, "failed fetching packages");
+  msg("extracting fetched packages");
+  if (!pkin_queue_extract_start(&extractq))
+    die(111, "failed extracting packages");
 }
 
 void command_fetch(int argc, char **argv)

pkin/src/bin/pkin.c

@@ -5,6 +5,7 @@
 #include <string.h>
 
 #include "../pkin.h"
+#include "../pkin_internal.h"
 #include "command.h"
 
 #define dienoarg(OPT)                     \
@@ -13,6 +14,7 @@
     dieusage();                           \
   } while(0)
 
+int pkin_extractjobs = 5;
 int pkin_verbose = 0;
 array pkin_repos;
 
@@ -61,7 +63,7 @@ static void dieusage()
 int main(int argc, char **argv)
 {
   int i;
-  errmsg_iam(*argv);
+  errmsg_iam("pkin");
   for (i = 1; i < argc; i++) {
     if (argv[i][0] != '-') break;
     if (argv[i][1] == '-' && !argv[i][2]) break;
@@ -82,6 +84,9 @@ int main(int argc, char **argv)
   if (argc == 0) dieusage();
   env_import_repos();
 
+  if (!pkin__mkpath(PATH_CACHE, 0755))
+    die(111, "failed creating cache ", PATH_CACHE);
+
   if (str_equal(*argv, "search"))
     command_search(argc, argv);
   else if (str_equal(*argv, "fetch"))

pkin/src/pkin.h

@@ -17,6 +17,7 @@
 #define PKIN_PKGMETA_URL_MAX 2048
 
 extern int pkin_verbose;
+extern int pkin_extractjobs;
 extern array pkin_repos; /* of char * */
 
 typedef struct {
@@ -49,17 +50,16 @@ void pkin_repo_iter_init(pkin_repo_iter *);
 int pkin_repo_iter_read(pkin_repo_iter *, pkin_pkgmeta *res);
 
 typedef struct {
-  buffer writeb;
-  char writebuf[1024];
+  buffer b;
+  char buf[1024];
   int pipefd[2];
-} pkin_fetch_queue;
+} pkin_queue;
+int pkin_queue_prepare(pkin_queue *queue);
 
-int pkin_fetch_prepare(pkin_fetch_queue *queue);
-void pkin_fetch_add(pkin_fetch_queue *queue, char *url, char *output);
-/*
- * downloads the binary package from url to the given
- * directory and writes the resulting path to rest
- */
-int pkin_fetch_start(pkin_fetch_queue *queue);
+void pkin_queue_fetch_add(pkin_queue *fetchqueue, char *url, char *output);
+int pkin_queue_fetch_start(pkin_queue *fetchqueue);
+
+void pkin_queue_extract_add(pkin_queue *extractqueue, char *src, char *dest);
+int pkin_queue_extract_start(pkin_queue *extractqueue);
 
 #endif

pkin/src/pkin/pkin_queue_extract_add.c

@@ -0,0 +1,9 @@
+#include "../pkin.h"
+
+void pkin_queue_extract_add(pkin_queue *queue, char *src, char *dest)
+{
+  buffer_puts(&queue->b, src);
+  buffer_puts(&queue->b, " ");
+  buffer_puts(&queue->b, dest);
+  buffer_puts(&queue->b, "\n");
+}

pkin/src/pkin/pkin_queue_extract_start.c

@@ -0,0 +1,43 @@
+#include <libowfat/errmsg.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#include <alloca.h>
+
+#include "../pkin_internal.h"
+#include "../pkin.h"
+
+/*
+ * given a queue of files to extract
+ * and the destination for the extracted file,
+ * this shall extract it in parallell.
+ * 
+ * basically:
+ *  spawn n extraction processes that consume queue
+ *  where n is pkin_extractjobs.
+ */
+
+int pkin_queue_extract_start(pkin_queue *queue)
+{
+  pid_t *pids;
+  int pids_len;
+  char **cmd;
+
+  if (buffer_flush(&queue->b) < 0) {
+    carpsys("write to pipe");
+    return 0;
+  }
+  close(queue->pipefd[1]);
+
+  pids = alloca(pkin_extractjobs);
+  pids_len = 0;
+  cmd = (char *[]){PATH_BIN_INTERNAL"/extraction-worker", NULL};
+  for (pids_len = 0; pids_len < pkin_extractjobs; pids_len++) {
+    pids[pids_len] = pkin__spawn(cmd, queue->pipefd[0], 1, 2);
+    /* TODO improve error handling(?) */
+    if (pids[pids_len] < 0) return 0;
+  }
+
+  for (pids_len = 0; pids_len < pkin_extractjobs; pids_len++)
+    waitpid(pids[pids_len], NULL, 0);
+  return 1;
+}

pkin/src/pkin/pkin_queue_fetch_add.c

@@ -0,0 +1,10 @@
+#include "../pkin.h"
+
+void pkin_queue_fetch_add(pkin_queue *queue, char *url, char *output)
+{
+  buffer_puts(&queue->b, url);
+  buffer_puts(&queue->b, "\n out=");
+  buffer_puts(&queue->b, output);
+  buffer_puts(&queue->b, "\n user-agent=pkin <https://git.lab0x13.site/kento2/pkin>");
+  buffer_puts(&queue->b, "\n");
+}

pkin/src/pkin/pkin_queue_fetch_start.c

@@ -0,0 +1,42 @@
+#include <libowfat/errmsg.h>
+#include <libowfat/open.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#include "../pkin.h"
+#include "../pkin_internal.h"
+
+int pkin_queue_fetch_start(pkin_queue *queue)
+{
+  char **cmd;
+  pid_t pid;
+  int status;
+  if (buffer_flush(&queue->b) < 0) {
+    carpsys("write to pipe");
+    return 0;
+  }
+  close(queue->pipefd[1]);
+
+  /* we use aria2c to process our queue using the -i option.
+   * aria2c logs errors to stdout(wtf???) and requires these options 
+   * to not flood the terminal. It will still always send the
+   * "Download Results"-report. */
+  cmd = (char *[]){"aria2c", "--console-log-level=warn",
+      "--enable-color=false", "--allow-overwrite=true", "-i", "-", NULL};
+  pid = pkin__spawn(cmd, queue->pipefd[0], 1, 2);
+  if (pid < 0) {
+    carp("failed spawning aria2c to fetch files");
+    return 0;
+  }
+
+  if (waitpid(pid, &status, 0) < 0) {
+    carp("waitpid");
+    return 0;
+  }
+  if (WEXITSTATUS(status) != 0) {
+    buffer_puts(buffer_2, "aria2c exited with non-zero exitcode: ");
+    buffer_putlong(buffer_2, WEXITSTATUS(status));
+    buffer_putsflush(buffer_2, "\n");
+    return 0;
+  }
+  return 1;
+}

pkin/src/pkin/pkin_queue_prepare.c

@@ -3,13 +3,13 @@
 
 #include "../pkin.h"
 
-int pkin_fetch_prepare(pkin_fetch_queue *queue)
+int pkin_queue_prepare(pkin_queue *queue)
 {
   if (pipe(queue->pipefd) < 0) {
     carpsys("pipe");
     return 0;
   }
-  buffer_init_write(&queue->writeb, queue->pipefd[1],
-      queue->writebuf, sizeof(queue->writebuf));
+  buffer_init_write(&queue->b, queue->pipefd[1],
+      queue->buf, sizeof(queue->buf));
   return 1;
 }

pkin/src/pkin_internal.h

@@ -5,4 +5,10 @@
 
 pid_t pkin__spawn(char *args[], int infd, int outfd, int errfd);
 
+/*
+ * makes sure path exist.
+ * return: 1 on success, 0 on error.
+ */
+int pkin__mkpath(const char *path, mode_t mode);
+
 #endif

pkin/src/pkin_internal/pkin__mkpath.c

@@ -0,0 +1,41 @@
+#include <libowfat/byte.h>
+#include <libowfat/str.h>
+#include <libowfat/byte.h>
+#include <libowfat/errmsg.h>
+#include <sys/stat.h>
+#include <alloca.h>
+#include <errno.h>
+
+#include "../pkin_internal.h"
+#include "src/pkin.h"
+
+int pkin__mkpath(const char *_path, mode_t mode)
+{
+  char *path;
+  size_t pathlen, l, off;
+  off = 0;
+  pathlen = str_len(_path);
+  /* copy path to make it writable */
+  path = alloca(pathlen + 1);
+  byte_copy(path, pathlen + 1, _path);
+
+  if (pkin_verbose)
+    msg("creating directory ", path);
+  for (;;) {
+    l = str_chr(path + off, '/');
+    if (l == 0) {
+      off++;
+      continue;
+    }
+    (path+off)[l] = 0;
+    if (mkdir(path, mode) < 0 && errno != EEXIST) {
+      carpsys("mkdir ", path);
+      return 0;
+    }
+    if (l == pathlen - off) break;
+    (path+off)[l] = '/';
+    off += l + 1;
+  }
+
+  return 1;
+}

pkin/src/pkin_internal/pkin__spawn.c

@@ -28,9 +28,7 @@ pid_t pkin__spawn(char *args[], int infd, int outfd, int errfd)
     if (dup2(infd, 0)<0
      || dup2(outfd, 1)<0
      || dup2(errfd, 2)<0
-    ) {
-      diesys(111, "dup2");
-    }
+    ) diesys(111, "dup2");
     if (infd != 0) close(infd);
     if (outfd != 1) close(outfd);
     if (errfd != 2) close(errfd);