From 7693240e21a861aa8c4ca4bee9da9d3de4c3fc39 Mon Sep 17 00:00:00 2001
From: Serply <59339358+googio@users.noreply.github.com>
Date: Mon, 10 Jun 2024 18:17:41 -0400
Subject: [PATCH] 1646-added serply (#1647)
* added serply search api
* undo remove of new line
---------
Co-authored-by: teampen <136991215+teampen@users.noreply.github.com>
---
docker/.env.example | 5 +-
docker/HOW_TO_USE_DOCKER.md | 2 +-
.../SearchProviderOptions/index.jsx | 35 +++++++++
.../WebSearchSelection/icons/serply.png | Bin 0 -> 4553 bytes
.../AgentConfig/WebSearchSelection/index.jsx | 10 +++
server/.env.example | 3 +
server/models/systemSettings.js | 3 +-
.../agents/aibitat/plugins/web-browsing.js | 69 ++++++++++++++++++
server/utils/helpers/updateENV.js | 5 ++
9 files changed, 129 insertions(+), 3 deletions(-)
create mode 100644 frontend/src/pages/WorkspaceSettings/AgentConfig/WebSearchSelection/icons/serply.png
diff --git a/docker/.env.example b/docker/.env.example
index 174a9d692..a38b4c5a2 100644
--- a/docker/.env.example
+++ b/docker/.env.example
@@ -241,4 +241,7 @@ GID='1000'
# AGENT_SERPER_DEV_KEY=
#------ Bing Search ----------- https://portal.azure.com/
-# AGENT_BING_SEARCH_API_KEY=
\ No newline at end of file
+# AGENT_BING_SEARCH_API_KEY=
+
+#------ Serply.io ----------- https://serply.io/
+# AGENT_SERPLY_API_KEY=
diff --git a/docker/HOW_TO_USE_DOCKER.md b/docker/HOW_TO_USE_DOCKER.md
index fed16a274..f570dce90 100644
--- a/docker/HOW_TO_USE_DOCKER.md
+++ b/docker/HOW_TO_USE_DOCKER.md
@@ -89,7 +89,6 @@ mintplexlabs/anythingllm;
<tr>
<td> Docker Compose</td>
<td>
-
version: '3.8'
services:
anythingllm:
@@ -116,6 +115,7 @@ mintplexlabs/anythingllm;
- TTS_PROVIDER=native
- PASSWORDMINCHAR=8
- AGENT_SERPER_DEV_KEY="SERPER DEV API KEY"
+ - AGENT_SERPLY_API_KEY="Serply.io API KEY"
volumes:
- anythingllm_storage:/app/server/storage
restart: always
diff --git a/frontend/src/pages/WorkspaceSettings/AgentConfig/WebSearchSelection/SearchProviderOptions/index.jsx b/frontend/src/pages/WorkspaceSettings/AgentConfig/WebSearchSelection/SearchProviderOptions/index.jsx
index 3d579ed43..58ceb8447 100644
--- a/frontend/src/pages/WorkspaceSettings/AgentConfig/WebSearchSelection/SearchProviderOptions/index.jsx
+++ b/frontend/src/pages/WorkspaceSettings/AgentConfig/WebSearchSelection/SearchProviderOptions/index.jsx
@@ -147,3 +147,38 @@ export function BingSearchOptions({ settings }) {
</>
);
}
+
+export function SerplySearchOptions({ settings }) {
+ return (
+ <>
+ <p className="text-sm text-white/60 my-2">
+ You can get a free API key{" "}
+ <a
+ href="https://serply.io"
+ target="_blank"
+ rel="noreferrer"
+ className="text-blue-300 underline"
+ >
+ from Serply.io.
+ </a>
+ </p>
+ <div className="flex gap-x-4">
+ <div className="flex flex-col w-60">
+ <label className="text-white text-sm font-semibold block mb-4">
+ API Key
+ </label>
+ <input
+ type="password"
+ name="env::AgentSerplyApiKey"
+ className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+ placeholder="Serply API Key"
+ defaultValue={settings?.AgentSerplyApiKey ? "*".repeat(20) : ""}
+ required={true}
+ autoComplete="off"
+ spellCheck={false}
+ />
+ </div>
+ </div>
+ </>
+ );
+}
diff --git a/frontend/src/pages/WorkspaceSettings/AgentConfig/WebSearchSelection/icons/serply.png b/frontend/src/pages/WorkspaceSettings/AgentConfig/WebSearchSelection/icons/serply.png
new file mode 100644
index 0000000000000000000000000000000000000000..8ac0ef7c00d094dfa8f85ad48fd02eb2fc209742
GIT binary patch
literal 4553
zcmds5=Q|q?7d48iQmTroQKQsIsM>o}jT%L$P1TB3v10d!8m&<)Hb1p#gczwkLaZ7k
zYJ?D@YKPh}-o9VoKjHmw?s@J#U+#zdoaebG-pJqu{jCSL$jHd(b+k2&|1#pgxJmuD
zeo9U9|4X#*v@Lwe$mp2=3v#lYJhs0>a$n;Y>SQ&e4>$fcH(b>8)yT-|lIboSsL05e
zY;`o%OrhjE3p9SFATDYmFrTa_y6PS|(`T+=rWvWCXj|ADqxt8mMrvIdk9mrkc^Lmn
zygj<+^DKcjrabR&X)~wu_Nmizs_STHw_l@PX2ISH4=((<9L@F}SQ&YH@GIox(mN==
zP=`Dv=0dIHNla!#A2GFw&358GT5s+RRLANz(CrgJz*C~FMC|`ZgsVb%>H=S6rKU0(
zH48`PG^yHu5;|SKL-_-}<YK&3maA0k(DLr8`x6T=e*6(16ShQ-k&%qo?f^V%-vT8M
zEKhnq5gBOT5_m_mWA)YSQryO<CqKf5yt7L+`Y_~!SncD{e1+KF+;}tGEf(Rpy8?Q=
zsDohWNU92njhsG*NBoPj(1}%Ru+&u@!KB)gQy~yjI~~J^I<SE1$IJM5Pd&Y13Z)3A
zqWcGea-Xgt9Bxbo;2@HP3>l?d^6#-xB-K?qWqJg*vr>hNECqNPn#y~*>ce;zIfiy`
z38K{OSgL2Riou$;>Z4)~LG^r_-z!b2)Q%qT)!Op$2;J1IA%;OY#q*UC8-_E<rm|a6
z;Q|yxAXV)}wVPnB(z1<<rOM$<pkv5|^h0?l<FvQ6IdCJk=L21_@!F6p{tK0Ei$p-s
z%DMkGzMX?~vZJA{9w{$q!o+2JgGI!wm%H$ak4p?BOq%}YLs@NJGj!S#dUg;8p1ON=
zb@k16ouT9M7svKY2jw&f*y;M?3!6mny~wDj#4wa+Vt)$Xi-ZUgiByz*E>~q*`6e{X
zQ2(aqr{sRTQph_A!k0Su>;2^la|9qq@#x;cuMs*!&jln^e-c;y-0|<?<S!cKNwdm<
z$-#CZor{i*aP1V{Q<LOfqGnR`wAmb@{GCgWcC=pv!y`&+4(5lupUFfmYZ*VWiuEkC
z26FB)msp-=;U%E^rI|+7VO_J${#*BzE%yVz7s`yCqWJ2(&x>D-6+k=o>=MLDP7n&)
z)?gNI!(7<D{}YzeIC&=7uwS<y_Z#$fbBEW?T~86vPkTs(z>cm5Quo~h?61k&lzgeB
zz2@U)efD?=JyflGsMD})SD|S0SbB|QTP3EucPRpG+951Hk}eMf;T9=ho{v}f@U3VL
zCgj!9tpmGYh|Z=tuE|zZKVd|vdT3%YrOk^f)f@oQuiqLL1K6}g6EtF%7n*$~>1(ug
zbh<yTiHLg)Fy_4~8vNLRY;OtdPuQ50q~E>92g{owEI(X@k(PZ~${HG;$!a(^I`{WE
z9LA*tbdMx8G(=Ii#nG-{6mHkd`uq1~-Q)N~Krr-WPr5ACl#LW1b3I!Fjvh35;Pcl0
zBvn=SUA2e6o2V)*rj;YYxeA7C^mqljxW)R~uRZ)W+>lMK^I0;)sWKqpyF4E)Zr7$&
z!u=fg&6kvf4hz_sk0%Enn$TPe<=M3I6WFdXuRSi-T?~|799y(Hiy0KilA90Go+k=R
z?#k80XolOmHPak!#p<yFdpHGj(^dWa8Wy$yHf{XFX`mr`xqn2Z!1HX$)gtNTki}qp
z3BQ!&3Hi$>qrh#sBIS!yxD~u<u(Z`Ldm=93qh|F__^Tpys_N^^rBFD)3Ac=zYsY5{
zn<p@~NTU7z{L*iOTW)H^GRQS{-dJ)tJDZVMnb_M^?_uCnU<;$sD>ul0dVW7?UiYPG
zVi%xwXJ-yI*3bZXUQ-d|sY9RF6evT%p|pm*nvhF_bfrf7?}q##YL@C^V*Piw<z6P_
zzaOzvs%ZCG>_|*3^we~p`jU00tbx~Yku54s(9p8I;3A?KvNkT<LSF@f$Nam*a{<|8
zo$8+RYM6k*$3Fa7uUlFOipy2#-;g)4?X>TX6hhzwD>m&1X@Hz^F4`lrW%}-#Sa|IC
zxKZBOW#INq9i>7iiD7VX(2Nw`cID_eqK?)|v}PD*u0mU(Y?VTf)-u^~tE(jFLNGQd
zt`=t}pCy=NmHa36!-pb$oGVnL<~Blmwh_HS)P3zS_0?vW&4%%p8MLQ2UW*2knxM#E
z`O{X}?~#9vQ`g9t_s=#tOcHnOtz;FiqX^^Hk$g#-y=sGar%#VF{VGquQ#Fa(+JeZ&
z7I|$N__kv*yOcw!FrE6r>fmv6awYSojen}LMPt>}v5RS}>&{$bmb_akNKnrJ>7)ny
zVtKTw_1XGSe*baRQR8jI>GI+==9^4E-^{XRh9eVV6SARc^)cb2`acv@OZ3gsZEk`Q
z0l|xkCs8hLs1|F^na;%u_`&*^_YI?%fU>cyl#Q(c^6SGi!)>sv8@=GmoH(S@@LYs9
zORWPsc7>R%;y`NhWXF|)G7T~$37+#M5*P#MBpcW8rVCO4txv}ae5xh8jW>7YIt*_@
z2|Ie(AAgmtW_Bkf?Z}24dti)mMM`WlUfm7q$WAxk2MZybw#?XZC8G;R3kKEeLoVaZ
z?yOL~n)cWczn!W{($7k({@uGmG$+|yN)okp5pCrF0txgPVcVSQl$c!sm<l=GZp7g$
zeFf7^Ibrvv<RFLQmD8a<<EpK&ZPLP`i<KR9l|y&ISnq49X^Z^U0Dm@XNw@Bh<4vc$
zP*`eOYTb)=nZN;3f(N@3D(tew-^`9%AsAaxC#Lh{uW2*?UBxqqu=*~|wTgPVw5d_)
zwK2`!T;njbcO)lX7hEX)!}CY89&7p&D-3`LiT5>)+%_rg?WL_7&;?so*?baPX^F3*
zG*_KMX49HvjvufLp+UxAY_%N_ci97?ss1QjXfTT<c3vKH%nNWX9l1|iWZW<&*}Y1P
zvZn7fnA#kg>VLU;Vk;qW^X4Ibgtl3x4e<F6WPd{7bc;A`>W;$sxVxm{kH;_?j~}k8
zUitTK4P*lCFo20>pJxReolYY2Dd~A7T!RBuJ!uU23TZUPcK5~1L~)lCro*<&OeFT_
z*%Ik1s|JWpt&topZq=B@@2fLnc1^c6K(zj#{D!8LPJ%ENzgdxexj26gn5@@Rv5rY8
ze!<b`X_gTG@s3LGxH<I-GG|`=k2!aVOw7CQ6qFQGd|NQg-W3{jSJ6dK!`Ni!<j<ca
zmkj2Pt-_XB&+F;+LJmpr(~+>{Z$~r!_0A*20lws}Tt#ru<pt%^F}z=yC%;^!`pH}X
z=bU45j$X1k<!bC5ZfQVvCByxjp5~b?Yuo@CFaMM~#=$2O&1rMf#zj3JwlOmKkMjGB
zfL}%4@mnN*mX##%q_NSmZtE2hc?m>K&3t(wo0f(t(b9BYm8$(6crK~}Ci}#(H5BlQ
zVe_qF(0zR9Sqe^@Mtk%fyOB3SUDSWtO=|aStxW$^GttMA%Y(L|;eJF=5rWBpwnh{t
z=PLOdU!r4@U{Xx3=Jhpnc%}DUrb5Y72790MeMfAPc0cH++Duhw1ltcANSkG4ol~}h
zq@JN(KeAMBA5Feh=|vlqR;Ewz<oYn(`O>0<Q{`h`li^5yxfQng5m?a}11*K6s}8YM
zDDql0X_g1<zzak~z5(o7JVM!!=T#0XS^<O%ow!d`8P#U+9P}O#j-$#wqPi=Xtw?lc
zf1_7s`}eA>o{jLcO%w&cpii@Da4TBN!dB*nxd25Dd<S1<i&`S{ZqAm)&<XUyNJ?;L
zIdt2zCilT&BDM4-rLnZPx&eo>AKPf5Lgs!W_O0)Fr^`tZwSueM+EUKTI}8DStpTs=
zgXz8S(AqfL;2+-l1(BIFccg<)VPeF-=o^kx=?&YklBlOvBejG4kqGTDIp<ZOHsv?b
z5}{57t`cKH0P9+^Tv2(D^AAtl1INaK)IzwyLd&T^zN#~<n*|9mOsWJz@8OqzrD)f@
zrri+CD>Nd0I6vBQjLNYVPdlQTEbI92%2A8MzdZVQJG(yXX%KS37mY^0lD$y76hyl{
zm2UZc`Jq<yw^J2pbKSBL%wF&D2poKRxFHF?btoQwxn^EHozARod1n0ACf!{??1$&>
zh@#-c2NhK{fTFX52Tl_yg$x{ll(?j}Rghr7^*_L1rvcrvdWC=u4;EW1`Ug_a@R>lU
zwCVkak%|&wK7m6V;?3fKEl(uRId@?=Sc=@|w7igb(4|&kU5aEXTT$UgSuOD6+@E7d
z*>dk07c+#qea<Pu#dCYWp6I%Z*1N(a-I7erB+ZpVj-GAq#j0r;zO}UCvPdaJp>m{~
zY)V!E#ePQc&F^lLi5z4egDjgH1KN(rURFgeVY-hVQU4k!z5Put_k1_Wb<4gqRl>1M
zi}P*Su=F?e%_SGoVVxBjpV&}d(8)C2HzVZGNv4_e0k*Q@U^Q((f3^L2Be)df5Zj$w
zQuui`9g)ZH53(SL*y>pS)<ZMRKc|{eSr2cVTz!%T5@X%*qtOZOGQsD3%mH^Tq!^c8
z%XmFxXJ0#X9}<8Zw*|gZ;3Xa1#v2(VCdf*5KN7RMCZ`nN%9ky_Px@Bnj`CtaO?%ID
zRpN^0*YbZE+i<O-G-8snZ(OJ1IpYG%u4=`<$9+Pd&dekjcQaW;jIW=q*MS!-9m|A7
z`mncuQh-RK&D;auhu-UEh*a(=(dj9lcCmp&Ugw}(lcqM;T{#ypgpjo1Cef0hh`~2)
z%HX$Jv7@+zK0}9}=rV|LWBl=MLXOxTi<7Wuw-}DM;%rh|sg|{J(L>5C=8W~gLwueE
z`F5{ad5kc-ULfU_C<i%>lK=(DY}807nwGs}$C`2i%rWklraF(>eS-$zT1cWiK|S&Z
zld$F2in^zIFbR1LOrbeW)IQ~8dly)r#%yZP74&DFgO8VVva8Y4cf&Dce_6nE9q9}^
z$+QSAvWt+C#t(w6_LpZ3Rp{8=nloIL;I{B!sg>8H)aj6W;g5nCv9*=^1E%l26`mdt
zGECK;Y^-f)yTzAME(4n1D%59TI43sJ6;c*J$8|^y8U&emd5wEk?FqSP_21jgXoc0M
z^0kXz6M1>U20<^J0Zwrq1H*`=z+4uvVbv%&ikg1OT@G3Gy|ZO={NWi>NjZI?hw6L+
zJLZ-m&Yx#uvqSvfPD4SL?~sx;`md8@;YA(oC7<_7N=uh7&i({+vp|w_{jhoM*Q3mG
z%=jE8B5*zbAIOkWi6QY<@9X67=6DeZe_Qw&nsGvmI^5x6Snc`_MHEFM$Wg>B{4KWi
z&JKw;_QqwAfbsFR<`C=k4JU*~xk1Hg^wA|)Dpc`aMs{4BJoNEn%<D$8f9=V!6|d0W
zM84S<#gaIPO6MkK1y3uv+z&oJ*S4*nKlPM@UFc<~%?ypHre{BFryl^?DX(s*uoV@*
zA(;Qm6K9=LEQS!6gljr_z8gvs)4pDl{VWLVG4mjM<qXk>##{HP<mvzae*OOwV4MR(
Yt;erp({yhCO<u@!o*QV^sM|;W2bv!b9{>OV
literal 0
HcmV?d00001
diff --git a/frontend/src/pages/WorkspaceSettings/AgentConfig/WebSearchSelection/index.jsx b/frontend/src/pages/WorkspaceSettings/AgentConfig/WebSearchSelection/index.jsx
index 8e8f054ea..0bbc053ab 100644
--- a/frontend/src/pages/WorkspaceSettings/AgentConfig/WebSearchSelection/index.jsx
+++ b/frontend/src/pages/WorkspaceSettings/AgentConfig/WebSearchSelection/index.jsx
@@ -3,12 +3,14 @@ import AnythingLLMIcon from "@/media/logo/anything-llm-icon.png";
import GoogleSearchIcon from "./icons/google.png";
import SerperDotDevIcon from "./icons/serper.png";
import BingSearchIcon from "./icons/bing.png";
+import SerplySearchIcon from "./icons/serply.png"
import { CaretUpDown, MagnifyingGlass, X } from "@phosphor-icons/react";
import SearchProviderItem from "./SearchProviderItem";
import {
SerperDotDevOptions,
GoogleSearchOptions,
BingSearchOptions,
+ SerplySearchOptions
} from "./SearchProviderOptions";
const SEARCH_PROVIDERS = [
@@ -44,6 +46,14 @@ const SEARCH_PROVIDERS = [
description:
"Web search powered by the Bing Search API. Free for 1000 queries per month.",
},
+ {
+ name: "Serply.io",
+ value: "serply-engine",
+ logo: SerplySearchIcon,
+ options: (settings) => <SerplySearchOptions settings={settings} />,
+ description:
+ "Serply.io web-search. Free account with a 100 calls/month forever.",
+ },
];
export default function AgentWebSearchSelection({
diff --git a/server/.env.example b/server/.env.example
index 6148d594f..a88a8a039 100644
--- a/server/.env.example
+++ b/server/.env.example
@@ -238,3 +238,6 @@ TTS_PROVIDER="native"
#------ Bing Search ----------- https://portal.azure.com/
# AGENT_BING_SEARCH_API_KEY=
+
+#------ Serply.io ----------- https://serply.io/
+# AGENT_SERPLY_API_KEY=
diff --git a/server/models/systemSettings.js b/server/models/systemSettings.js
index ac0523198..3cb0456f4 100644
--- a/server/models/systemSettings.js
+++ b/server/models/systemSettings.js
@@ -71,7 +71,7 @@ const SystemSettings = {
try {
if (update === "none") return null;
if (
- !["google-search-engine", "serper-dot-dev", "bing-search"].includes(
+ !["google-search-engine", "serper-dot-dev", "bing-search", "serply-engine"].includes(
update
)
)
@@ -176,6 +176,7 @@ const SystemSettings = {
AgentGoogleSearchEngineKey: process.env.AGENT_GSE_KEY || null,
AgentSerperApiKey: process.env.AGENT_SERPER_DEV_KEY || null,
AgentBingSearchApiKey: process.env.AGENT_BING_SEARCH_API_KEY || null,
+ AgentSerplyApiKey: process.env.AGENT_SERPLY_API_KEY || null,
};
},
diff --git a/server/utils/agents/aibitat/plugins/web-browsing.js b/server/utils/agents/aibitat/plugins/web-browsing.js
index b30688f17..00b004cbc 100644
--- a/server/utils/agents/aibitat/plugins/web-browsing.js
+++ b/server/utils/agents/aibitat/plugins/web-browsing.js
@@ -68,6 +68,9 @@ const webBrowsing = {
case "bing-search":
engine = "_bingWebSearch";
break;
+ case "serply-engine":
+ engine = "_serplyEngine";
+ break;
default:
engine = "_googleSearchEngine";
}
@@ -218,6 +221,72 @@ const webBrowsing = {
return `No information was found online for the search query.`;
return JSON.stringify(searchResponse);
},
+ _serplyEngine: async function (query, language = "en", hl = "us", limit = 100, device_type = "desktop", proxy_location = "US") {
+ // query (str): The query to search for
+ // hl (str): Host Language code to display results in (reference https://developers.google.com/custom-search/docs/xml_results?hl=en#wsInterfaceLanguages)
+ // limit (int): The maximum number of results to return [10-100, defaults to 100]
+ // device_type: get results based on desktop/mobile (defaults to desktop)
+
+ if (!process.env.AGENT_SERPLY_API_KEY) {
+ this.super.introspect(
+ `${this.caller}: I can't use Serply.io searching because the user has not defined the required API key.\nVisit: https://serply.io to create the API key for free.`
+ );
+ return `Search is disabled and no content was found. This functionality is disabled because the user has not set it up yet.`;
+ }
+
+ this.super.introspect(
+ `${this.caller}: Using Serply to search for "${
+ query.length > 100 ? `${query.slice(0, 100)}...` : query
+ }"`
+ );
+
+ const params = new URLSearchParams({
+ q: query,
+ language: language,
+ hl,
+ gl: proxy_location.toUpperCase()
+ })
+ const url = `https://api.serply.io/v1/search/${params.toString()}`
+ const { response, error } = await fetch(
+ url,
+ {
+ method: "GET",
+ headers: {
+ "X-API-KEY": process.env.AGENT_SERPLY_API_KEY,
+ "Content-Type": "application/json",
+ "User-Agent": "anything-llm",
+ "X-Proxy-Location": proxy_location,
+ "X-User-Agent": device_type
+ }
+ }
+ )
+ .then((res) => res.json())
+ .then((data) => {
+ if (data?.message === "Unauthorized"){
+ return { response: null, error: "Unauthorized. Please double check your AGENT_SERPLY_API_KEY" };
+ }
+ return { response: data, error: null}
+ })
+ .catch((e) => {
+ return { response: null, error: e.message };
+ });
+ if (error)
+ return `There was an error searching for content. ${error}`;
+
+ const data = [];
+ response.results?.forEach((searchResult) => {
+ const { title, link, description } = searchResult;
+ data.push({
+ title,
+ link,
+ snippet: description,
+ });
+ });
+
+ if (data.length === 0)
+ return `No information was found online for the search query.`;
+ return JSON.stringify(data);
+ },
});
},
};
diff --git a/server/utils/helpers/updateENV.js b/server/utils/helpers/updateENV.js
index 1a0e710a9..3f2baf7e3 100644
--- a/server/utils/helpers/updateENV.js
+++ b/server/utils/helpers/updateENV.js
@@ -403,6 +403,10 @@ const KEY_MAPPING = {
envKey: "AGENT_BING_SEARCH_API_KEY",
checks: [],
},
+ AgentSerplyApiKey: {
+ envKey: "AGENT_SERPLY_API_KEY",
+ checks: [],
+ },
// TTS/STT Integration ENVS
TextToSpeechProvider: {
@@ -769,6 +773,7 @@ async function dumpENV() {
"AGENT_GSE_KEY",
"AGENT_SERPER_DEV_KEY",
"AGENT_BING_SEARCH_API_KEY",
+ "AGENT_SERPLY_API_KEY"
];
// Simple sanitization of each value to prevent ENV injection via newline or quote escaping.
--
GitLab