【Go开源库】微软推出屏幕解析工具OmniParser ，AI的好搭档

// 示例: OmniParser 基础用法
package main

import (
"fmt"
"log"
)

// 假设 OmniParser 提供 Go 客户端
// type OmniParser struct{}
// type ScreenElement struct {
//     Type     string
//     Text     string
//     BBox     BoundingBox
//     Attrs    map[string]string
// }

func main() {
// 创建 OmniParser 客户端
 parser := OmniParser{}

// 加载截图
 screenshot := "screenshot.png"

// 解析屏幕元素
 elements, err := parser.Parse(screenshot)
if err != nil {
  log.Fatalf("解析失败: %v", err)
 }

// 输出识别结果
 fmt.Printf("识别到 %d 个 UI 元素:\n", len(elements))
for i, elem := range elements {
  fmt.Printf("%d. [%s] %s at %v\n",
   i+1, elem.Type, elem.Text, elem.BBox)
 }
}

// 示例: 将 OmniParser 结果集成到 LLM 流程
package main

import (
"encoding/json"
"fmt"
"log"
)

type UIAction struct {
 Target  string`json:"target"`
 Action  string`json:"action"`
 Value   string`json:"value,omitempty"`
}

func generateLLMPrompt(elements []ScreenElement) string {
 prompt := "基于以下 UI 元素，执行任务: 点击登录按钮\n\n"
 prompt += "可用的 UI 元素:\n"

for _, elem := range elements {
  prompt += fmt.Sprintf("- %s: %s\n", elem.Type, elem.Text)
 }

return prompt
}

func parseLLMResponse(response string) UIAction {
var action UIAction
 json.Unmarshal([]byte(response), &action)
return action
}

func main() {
 parser := OmniParser{}
 elements, _ := parser.Parse("login_screen.png")

// 生成 LLM Prompt
 prompt := generateLLMPrompt(elements)
 fmt.Printf("LLM Prompt:\n%s\n", prompt)

// 模拟 LLM 返回
 llmResponse := `{"target":"login_button","action":"click"}`
 action := parseLLMResponse(llmResponse)

// 执行操作
 fmt.Printf("执行操作: %s %s\n", action.Action, action.Target)
}

// 示例: 批量处理多个屏幕截图
package main

import (
"fmt"
"os"
"path/filepath"
"sync"
)

type ProcessResult struct {
 Filepath string
 Elements []ScreenElement
 Error    error
}

func processScreenshots(dir string) []ProcessResult {
 files, _ := filepath.Glob(filepath.Join(dir, "*.png"))

 results := make([]ProcessResult, len(files))
var wg sync.WaitGroup
var mu sync.Mutex

 parser := OmniParser{}

for i, file := range files {
  wg.Add(1)
gofunc(idx int, filename string) {
   defer wg.Done()
   elements, err := parser.Parse(filename)

   mu.Lock()
   results[idx] = ProcessResult{
    Filepath: filename,
    Elements: elements,
    Error:    err,
   }
   mu.Unlock()
  }(i, file)
 }

 wg.Wait()
return results
}

func main() {
 results := processScreenshots("./screenshots")

for _, result := range results {
if result.Error != nil {
   fmt.Printf("处理失败: %s - %v\n", result.Filepath, result.Error)
   continue
  }
  fmt.Printf("处理成功: %s - %d 个元素\n",
   result.Filepath, len(result.Elements))
 }
}