From 95ee663b284550aab25b244d7c352738458bd8cd Mon Sep 17 00:00:00 2001
From: itchyny <itchyny@cybozu.co.jp>
Date: Tue, 4 Mar 2025 19:59:41 +0900
Subject: [PATCH] improve performance of `unique` and `unique_by` (#3254)

Previously, `unique` and `unique_by` filters are implemented using
`group_by` and then `map(.[0])`. This commit re-implements in C, to
avoid unnecessary boxing of grouping, and improves the performance.
---
 src/builtin.c  | 20 ++++++++++++++++++++
 src/builtin.jq |  3 +--
 src/jv.h       |  3 ++-
 src/jv_aux.c   | 22 ++++++++++++++++++++++
 4 files changed, 45 insertions(+), 3 deletions(-)

diff --git a/src/builtin.c b/src/builtin.c
index ad14da25..df9dfb62 100644
--- a/src/builtin.c
+++ b/src/builtin.c
@@ -879,6 +879,24 @@ static jv f_group_by_impl(jq_state *jq, jv input, jv keys) {
   }
 }
 
+static jv f_unique(jq_state *jq, jv input) {
+  if (jv_get_kind(input) == JV_KIND_ARRAY) {
+    return jv_unique(input, jv_copy(input));
+  } else {
+    return type_error(input, "cannot be sorted, as it is not an array");
+  }
+}
+
+static jv f_unique_by_impl(jq_state *jq, jv input, jv keys) {
+  if (jv_get_kind(input) == JV_KIND_ARRAY &&
+      jv_get_kind(keys) == JV_KIND_ARRAY &&
+      jv_array_length(jv_copy(input)) == jv_array_length(jv_copy(keys))) {
+    return jv_unique(input, keys);
+  } else {
+    return type_error2(input, keys, "cannot be sorted, as they are not both arrays");
+  }
+}
+
 #ifdef HAVE_LIBONIG
 static int f_match_name_iter(const UChar* name, const UChar *name_end, int ngroups,
     int *groups, regex_t *reg, void *arg) {
@@ -1912,6 +1930,8 @@ BINOPS
   CFUNC(f_sort, "sort", 1),
   CFUNC(f_sort_by_impl, "_sort_by_impl", 2),
   CFUNC(f_group_by_impl, "_group_by_impl", 2),
+  CFUNC(f_unique, "unique", 1),
+  CFUNC(f_unique_by_impl, "_unique_by_impl", 2),
   CFUNC(f_bsearch, "bsearch", 2),
   CFUNC(f_min, "min", 1),
   CFUNC(f_max, "max", 1),
diff --git a/src/builtin.jq b/src/builtin.jq
index bb32c515..ba12d4e5 100644
--- a/src/builtin.jq
+++ b/src/builtin.jq
@@ -4,8 +4,7 @@ def map(f): [.[] | f];
 def select(f): if f then . else empty end;
 def sort_by(f): _sort_by_impl(map([f]));
 def group_by(f): _group_by_impl(map([f]));
-def unique: group_by(.) | map(.[0]);
-def unique_by(f): group_by(f) | map(.[0]);
+def unique_by(f): _unique_by_impl(map([f]));
 def max_by(f): _max_by_impl(map([f]));
 def min_by(f): _min_by_impl(map([f]));
 def add(f): reduce f as $x (null; . + $x);
diff --git a/src/jv.h b/src/jv.h
index 83429569..5c3b7c54 100644
--- a/src/jv.h
+++ b/src/jv.h
@@ -265,8 +265,9 @@ jv jv_delpaths(jv, jv);
 jv jv_keys(jv /*object or array*/);
 jv jv_keys_unsorted(jv /*object or array*/);
 int jv_cmp(jv, jv);
-jv jv_group(jv, jv);
 jv jv_sort(jv, jv);
+jv jv_group(jv, jv);
+jv jv_unique(jv, jv);
 
 #ifdef __cplusplus
 }
diff --git a/src/jv_aux.c b/src/jv_aux.c
index a08ca88e..35da41bc 100644
--- a/src/jv_aux.c
+++ b/src/jv_aux.c
@@ -727,3 +727,25 @@ jv jv_group(jv objects, jv keys) {
   jv_mem_free(entries);
   return ret;
 }
+
+jv jv_unique(jv objects, jv keys) {
+  assert(jv_get_kind(objects) == JV_KIND_ARRAY);
+  assert(jv_get_kind(keys) == JV_KIND_ARRAY);
+  assert(jv_array_length(jv_copy(objects)) == jv_array_length(jv_copy(keys)));
+  int n = jv_array_length(jv_copy(objects));
+  struct sort_entry* entries = sort_items(objects, keys);
+  jv ret = jv_array();
+  jv curr_key = jv_invalid();
+  for (int i = 0; i < n; i++) {
+    if (jv_equal(jv_copy(curr_key), jv_copy(entries[i].key))) {
+      jv_free(entries[i].key);
+    } else {
+      jv_free(curr_key);
+      curr_key = entries[i].key;
+      ret = jv_array_append(ret, entries[i].object);
+    }
+  }
+  jv_free(curr_key);
+  jv_mem_free(entries);
+  return ret;
+}