Skip to content
Merged
Show file tree
Hide file tree
Changes from 11 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README(E).md
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# 首款直接接入autoglm api的移动端app

<div align="center">
<a href="README.md">中文版</a> | <span>English</span>
</div>
Expand Down
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# 首款直接接入AutoGLM API的移动端app

<div align="center">
<span>中文</span> | <a href="README(E).md">English</a>
</div>
Expand Down
3 changes: 2 additions & 1 deletion app/src/main/assets/packages/automatic_ui_base.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,11 @@
{
"name": "Automatic_ui_base",
"description": "提供基本的UI自动化工具,能够按照用户的要求帮助操作设备屏幕(如点击、滑动、输入等)。",
"enabledByDefault": true,
"tools": [
{
"name": "usage_advice",
"description": "UI自动化建议:\\n- 元素定位选项:\\n • 列表:使用index参数(例如,“点击索引为2的列表项”)\\n • 文本:使用bounds或partialMatch进行模糊匹配(例如,“点击包含‘登录’文字的按钮”)\\n- 操作链:组合多个操作以完成复杂任务(例如,“获取页面信息,然后点击元素”)\\n- 错误处理:如果操作失败,分析页面信息找出原因,并尝试其他方法。\\n- **组合调用(推荐)**:强烈建议在一次响应中组合调用2~3个真实存在的工具,例如依次调用tap → get_page_info,或 click_element → sleep → get_page_info,一次性输出完整的操作序列。软件会自动按顺序依次执行这些工具调用.",
"description": "UI自动化建议:\\n- 元素定位选项:\\n • 列表:使用index参数(例如,“点击索引为2的列表项”)\\n • 文本:使用bounds或partialMatch进行模糊匹配(例如,“点击包含‘登录’文字的按钮”)\\n- 操作链:组合多个操作以完成复杂任务(例如,“获取页面信息,然后点击元素”)\\n- 错误处理:如果操作失败,分析页面信息找出原因,并尝试其他方法。\\n- **组合调用(推荐)**:强烈建议在一次响应中组合调用2~3个真实存在的工具,例如依次调用tap → get_page_info,或 click_element → sleep → get_page_info,一次性输出完整的操作序列。软件会自动按顺序依次执行这些工具调用",
"parameters": []
},
{
Expand Down
2 changes: 1 addition & 1 deletion app/src/main/assets/packages/automatic_ui_subagent.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/* METADATA
{
"name": "Automatic_ui_subagent",
"description": "兼容AutoGLM,提供基于独立UI控制器模型(例如 autoglm-phone-9b)的高层UI自动化子代理工具,用于根据自然语言意图自动规划点击/输入/滑动等操作。仅在用户明确要求使用智能体控制时调用,默认场景请优先使用普通UI自动化工具包。",
"description": "兼容AutoGLM,提供基于独立UI控制器模型(例如 autoglm-phone-9b)的高层UI自动化子代理工具,用于根据自然语言意图自动规划并执行点击/输入/滑动等一系列界面操作。当用户提出需要帮忙完成某个界面操作任务(例如打开应用、搜索内容、在多个页面之间完成一套步骤)时,可以调用本包由子代理自动规划和执行具体步骤。",
"tools": [
{
"name": "run_subagent",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -203,15 +203,41 @@ class FileBindingService(context: Context) {

for ((op, start, end) in enrichedOps) {
AppLogger.d(TAG, "Applying ${op.action} at lines ${start + 1}-${end + 1}")


// Capture original segment before removal so we can preserve indentation if needed
val originalSegment = originalLines.subList(start, end + 1).toList()

// Remove the old lines
for (i in end downTo start) {
originalLines.removeAt(i)
}

// If it's a REPLACE, add the new lines
if (op.action == EditAction.REPLACE) {
originalLines.addAll(start, op.newContent.lines())
val newLinesRaw = op.newContent.lines()

// For simple single-line replacements, inherit the indentation of the original line
val newLines = if (originalSegment.isNotEmpty() &&
start == end &&
newLinesRaw.size == 1
) {
val originalFirstLine = originalSegment.first()
val indentPrefix = originalFirstLine.takeWhile { it == ' ' || it == '\t' }
val newLine = newLinesRaw.first()

if (indentPrefix.isNotEmpty() &&
!newLine.startsWith(" ") &&
!newLine.startsWith("\t")
) {
listOf(indentPrefix + newLine)
} else {
newLinesRaw
}
} else {
newLinesRaw
}

originalLines.addAll(start, newLines)
}
}

Expand Down Expand Up @@ -262,8 +288,8 @@ class FileBindingService(context: Context) {
i++
}

val normalizedOld = oldContent.trimTrailingNewline()
val normalizedNew = newContent.trimTrailingNewline()
val normalizedOld = oldContent.removeSuffix("\n").removeSuffix("\r")
val normalizedNew = newContent.removeSuffix("\n").removeSuffix("\r")

// Basic validation
if ((action == EditAction.REPLACE || action == EditAction.DELETE) && normalizedOld.isBlank()) {
Expand Down Expand Up @@ -296,6 +322,12 @@ class FileBindingService(context: Context) {
// --- 优化1:预计算与规范化 ---
AppLogger.d(TAG, "开始预计算与规范化...")
val normalizedOldContent = oldContent.replace(Regex("\\s+"), "")
val baseNgrams = buildNgrams(normalizedOldContent)
if (baseNgrams.isEmpty()) {
AppLogger.w(TAG, "OLD 块在去空白后过短,无法构建 n-gram,放弃匹配。")
return -1 to -1
}

val lineStartIndices = mutableListOf<Int>()
val normalizedOriginalContent = buildString {
originalLines.forEachIndexed { index, line ->
Expand Down Expand Up @@ -361,30 +393,30 @@ class FileBindingService(context: Context) {
val startCharIndex = lineStartIndices[i]
val endCharIndex = lineStartIndices[endLine]
val normalizedWindow =
normalizedOriginalContent.substring(startCharIndex, endCharIndex)
normalizedOriginalContent.substring(startCharIndex, endCharIndex)

localLcs++
val score = lcsRatio(normalizedOldContent, normalizedWindow)
val score = ngramSimilarity(baseNgrams, normalizedWindow)

if (score > localBestScore) {
localBestScore = score
localBestStart = i
localBestEnd = endLine - 1
val matchPercentage = (localBestScore * 100).toInt()
AppLogger.d(
TAG,
"并行块[$threadIndex] 发现更佳匹配: 行 ${i + 1}-$endLine, 相似度: $matchPercentage%"
TAG,
"并行块[$threadIndex] 发现更佳匹配: 行 ${i + 1}-$endLine, 相似度: $matchPercentage%"
)

if (localBestScore == 1.0) {
foundPerfectMatch.set(true)
AppLogger.d(TAG, "并行块[$threadIndex] 已找到100%匹配,提前结束该块搜索。")
return@Callable MatchSearchResult(
localBestScore,
localBestStart,
localBestEnd,
localWindows,
localLcs
localBestScore,
localBestStart,
localBestEnd,
localWindows,
localLcs
)
}
}
Expand Down Expand Up @@ -423,9 +455,12 @@ class FileBindingService(context: Context) {
val totalTime = (System.currentTimeMillis() - startTime) / 1000.0
val result = if (bestMatchScore > 0.9) {
val (start, end) = bestMatchRange
AppLogger.d(TAG, "匹配完成! 最佳匹配: 行 ${start + 1}-${end + 1}, 相似度: ${(bestMatchScore * 100).toInt()}%, " +
"总耗时: ${String.format("%.2f", totalTime)}s, " +
"总窗口数: $totalWindows, 总LCS计算: $lcsCalculations")
AppLogger.d(
TAG,
"匹配完成! 最佳匹配: 行 ${start + 1}-${end + 1}, 相似度: ${(bestMatchScore * 100).toInt()}%, " +
"总耗时: ${String.format("%.2f", totalTime)}s, " +
"总窗口数: $totalWindows, 总LCS计算: $lcsCalculations"
)
bestMatchRange
} else {
AppLogger.w(TAG, "未找到足够好的匹配 (最高相似度: ${(bestMatchScore * 100).toInt()}% < 90%)")
Expand All @@ -435,26 +470,22 @@ class FileBindingService(context: Context) {
return result
}

private fun lcsRatio(s1: String, s2: String): Double {
val len1 = s1.length
val len2 = s2.length
if (len1 == 0 || len2 == 0) return 0.0
private fun buildNgrams(s: String, n: Int = 3): Set<String> {
if (s.length < n) return emptySet()
return s.windowed(n, 1).toSet()
}

// 动态规划表
val dp = Array(len1 + 1) { IntArray(len2 + 1) }
private fun ngramSimilarity(baseNgrams: Set<String>, s2: String, n: Int = 3): Double {
if (baseNgrams.isEmpty() || s2.isEmpty()) return 0.0
if (s2.length < n) return 0.0

for (i in 1..len1) {
for (j in 1..len2) {
if (s1[i - 1] == s2[j - 1]) {
dp[i][j] = dp[i - 1][j - 1] + 1
} else {
dp[i][j] = maxOf(dp[i - 1][j], dp[i][j - 1])
}
}
}

val lcsLength = dp[len1][len2]
return (2.0 * lcsLength) / (len1 + len2)
val ngrams2 = s2.windowed(n, 1).toSet()
if (ngrams2.isEmpty()) return 0.0

val intersection = baseNgrams.intersect(ngrams2).size
val union = baseNgrams.size + ngrams2.size - intersection

return if (union == 0) 0.0 else intersection.toDouble() / union.toDouble()
}

private fun hasMultiplePerfectMatches(originalContent: String, oldContent: String): Boolean {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,13 +87,15 @@ object AIServiceFactory {

return when (config.apiProviderType) {
// OpenAI格式,支持原生和兼容OpenAI API的服务
ApiProviderType.OPENAI -> OpenAIProvider(config.apiEndpoint, apiKeyProvider, config.modelName, httpClient, customHeaders, config.apiProviderType, supportsVision, enableToolCall)
ApiProviderType.OPENAI,
ApiProviderType.OPENAI_GENERIC -> OpenAIProvider(config.apiEndpoint, apiKeyProvider, config.modelName, httpClient, customHeaders, config.apiProviderType, supportsVision, enableToolCall)

// Claude格式,支持Anthropic Claude系列
ApiProviderType.ANTHROPIC -> ClaudeProvider(config.apiEndpoint, apiKeyProvider, config.modelName, httpClient, customHeaders, config.apiProviderType, enableToolCall)

// Gemini格式,支持Google Gemini系列
ApiProviderType.GOOGLE -> GeminiProvider(config.apiEndpoint, apiKeyProvider, config.modelName, httpClient, customHeaders, config.apiProviderType, config.enableGoogleSearch, enableToolCall)
// Gemini格式,支持Google Gemini系列及通用Gemini端点
ApiProviderType.GOOGLE,
ApiProviderType.GEMINI_GENERIC -> GeminiProvider(config.apiEndpoint, apiKeyProvider, config.modelName, httpClient, customHeaders, config.apiProviderType, config.enableGoogleSearch, enableToolCall)

// LM Studio使用OpenAI兼容格式
ApiProviderType.LMSTUDIO -> OpenAIProvider(config.apiEndpoint, apiKeyProvider, config.modelName, httpClient, customHeaders, config.apiProviderType, supportsVision, enableToolCall)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,11 @@ object ModelListFetcher {

val modelsUrl =
when (apiProviderType) {
ApiProviderType.OPENAI -> "${extractBaseUrl(apiEndpoint)}/v1/models"
ApiProviderType.OPENAI,
ApiProviderType.OPENAI_GENERIC -> "${extractBaseUrl(apiEndpoint)}/v1/models"
ApiProviderType.ANTHROPIC -> "${extractBaseUrl(apiEndpoint)}/v1/models"
ApiProviderType.GOOGLE -> {
ApiProviderType.GOOGLE,
ApiProviderType.GEMINI_GENERIC -> {
// 对于Gemini API,直接使用提供的端点或默认端点
if (apiEndpoint.contains("generativelanguage.googleapis.com")) {
// 如果端点已经是模型列表URL,直接使用
Expand Down Expand Up @@ -153,7 +155,8 @@ object ModelListFetcher {

// 根据不同供应商添加不同的认证头
when (apiProviderType) {
ApiProviderType.GOOGLE -> {
ApiProviderType.GOOGLE,
ApiProviderType.GEMINI_GENERIC -> {
// Google Gemini API 使用 API 密钥作为查询参数
val urlWithKey =
if (modelsUrl.contains("?")) {
Expand Down Expand Up @@ -210,6 +213,7 @@ object ModelListFetcher {
try {
when (apiProviderType) {
ApiProviderType.OPENAI,
ApiProviderType.OPENAI_GENERIC,
ApiProviderType.DEEPSEEK,
ApiProviderType.MOONSHOT,
ApiProviderType.SILICONFLOW,
Expand All @@ -218,11 +222,10 @@ object ModelListFetcher {
ApiProviderType.INFINIAI,
ApiProviderType.ALIPAY_BAILING,
ApiProviderType.LMSTUDIO,
ApiProviderType.PPINFRA ->
parseOpenAIModelResponse(responseBody)
ApiProviderType.ANTHROPIC ->
parseAnthropicModelResponse(responseBody)
ApiProviderType.GOOGLE -> parseGoogleModelResponse(responseBody)
ApiProviderType.PPINFRA -> parseOpenAIModelResponse(responseBody)
ApiProviderType.ANTHROPIC -> parseAnthropicModelResponse(responseBody)
ApiProviderType.GOOGLE,
ApiProviderType.GEMINI_GENERIC -> parseGoogleModelResponse(responseBody)

// 其他提供商可能需要单独的解析方法
else -> parseOpenAIModelResponse(responseBody) // 默认尝试OpenAI格式
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1245,7 +1245,6 @@ open class OpenAIProvider(
val line = reader.readLine() ?: break

if (!line.startsWith("data:")) {
logLargeString("AIService", line, "【发送消息】收到非data行: ")
continue
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -131,9 +131,7 @@ object FunctionalPrompts {

操作指令及其作用如下:
- do(action="Launch", app="xxx")
Launch是启动目标app的操作,这比通过主屏幕导航更快。**参数 app 必须是Android的包名字符串,而不是中文名称或简称**,例如微信=com.tencent.mm,B站=tv.danmaku.bili。此操作完成后,您将自动收到结果状态的截图。
- do(action="List")
List是列出已安装应用及其Android包名的操作。你可以在不知道某个应用包名时先调用一次List,查看结果中的应用名称和对应包名,然后在后续的Launch动作中使用准确的包名。
Launch是启动目标app的操作,这比通过主屏幕导航更快。此操作完成后,您将自动收到结果状态的截图。
- do(action="Tap", element=[x,y])
Tap是点击操作,点击屏幕上的特定点。可用此操作点击按钮、选择项目、从主屏幕打开应用程序,或与任何可点击的用户界面元素进行交互。坐标系统从左上角 (0,0) 开始到右下角(999,999)结束。此操作完成后,您将自动收到结果状态的截图。
- do(action="Tap", element=[x,y], message="重要操作")
Expand Down
Loading