الصوت في تطبيق الهاتف المحمول: تعلم شاشات الاتصال وملء النماذج دون استخدام اليدين

كيفية تضمين واجهة صوتية في تطبيق الهاتف المحمول الخاص بك بسرعة وسهولة؟ وكيف تعلم المساعد كل ما يمكنه فعله؟ في المرة الأخيرة التي اتخذناها opensorsnoe تطبيق نمط الحياة Habitica وأظهرت كيفية إضافة مساعد ويجرحون النصي الصوتية الأساسية "للخروج من مربع" (توقعات الطقس التحديث والوقت). والآن دعنا ننتقل إلى مرحلة أكثر تقدمًا - سوف نتعلم كيفية الاتصال بشاشات معينة عن طريق الصوت ، وإجراء طلبات معقدة باستخدام NLU وملء النماذج باستخدام صوت داخل التطبيق.

(اقرأ الجزء الأول من البرنامج التعليمي)

, Habitica – : ,     . , , , , .

– . , , . AndroidManifest . PrefsActivity, , FixCharacterValuesActivity, , , , FullProfileActivity AboutActivity.

, , CustomSkill. -, , , response.action “changeView”. response.intent , – . :

class ChangeViewSkill(private val context: Context): CustomSkill<AimyboxRequest, AimyboxResponse> {

    override fun canHandle(response: AimyboxResponse) = response.action == "changeView"

    override suspend fun onResponse(
            response: AimyboxResponse,
            aimybox: Aimybox,
            defaultHandler: suspend (Response) -> Unit
    ) {
        val intent = when (response.intent) {
            "settings" -> Intent(context, PrefsActivity::class.java)
            "characteristics" -> Intent(context, FixCharacterValuesActivity::class.java)//
            "profile" -> Intent(context, FullProfileActivity::class.java)//
            "about" -> Intent(context, AboutActivity::class.java)
            else -> Intent(context, MainActivity::class.java)
        }
        intent.setFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
        aimybox.standby()
        context.startActivity(intent)
    }
}

:

 val dialogApi = AimyboxDialogApi(
                "YOUR KEY HERE", unitId,
            customSkills = linkedSetOf(ChangeView()))

 JAICF ( Just AI Kotlin).

https://github.com/just-ai/jaicf-jaicp-caila-template.

, JAICP (Just AI Conversational Platform) c Aimybox (SDK ), – connections. , . AimyboxConnection.

package com.justai.jaicf.template.connections
import com.justai.jaicf.channel.http.httpBotRouting
import com.justai.jaicf.channel.aimybox.AimyboxChannel
import io.ktor.routing.routing
import io.ktor.server.engine.embeddedServer
import io.ktor.server.netty.Netty
import com.justai.jaicf.template.templateBot

fun main() {
    embeddedServer(Netty, System.getenv("PORT")?.toInt() ?: 8080) {
        routing {
            httpBotRouting("/" to AimyboxChannel(templateBot))
        }
    }.start(wait = true)
}

, NLU-, NLU- Caila – app.jaicp.com, API conf/jaicp.properties. , app.jaicp.com.



NLU- – , , NLU.

. , . , , DATA , (settings, characteristics, .. ).

:

, . . , , , , views . .

JAICF.

, , - «» . .

:

, catchAll – , , . changeView, activators JAICP , actions – , Aimybox , .

views , Caila, action , , Aimybox , , . «». - .

        state("changeView") {
            activators {
                intent("changeView")
            }
            action {
                reactions.say("..." )
                var slot = ""
                activator.caila?.run {slot = slots["views"].toString()}
                reactions.aimybox?.response?.action = "changeView"
                reactions.aimybox?.response?.intent = slot

            }
        }

skills .

. ngrok, heroku. app.aimybox.com, , Aimylogic webhook URL. : , .

, , Try in Action.

, – , .

. . , .

! .

, , , - (- ), .

. response.action == "createTask", , response.intent. 

, , , , , TaskFormActivity, . .

class CreateTaskSkill(private val context: Context): CustomSkill<AimyboxRequest, AimyboxResponse> {


    override fun canHandle(response: AimyboxResponse) = response.action == "createTask"

    override suspend fun onResponse(
            response: AimyboxResponse,
            aimybox: Aimybox,
            defaultHandler: suspend (Response) -> Unit
    ) {
        val intent = Intent(context, TaskFormActivity::class.java)
        val additionalData = HashMap<String, Any>()
        val type = response.intent
        additionalData["viewed task type"] = when (type) {
            "habit" -> Task.TYPE_HABIT
            "daily" -> Task.TYPE_DAILY
            "todo" -> Task.TYPE_TODO
            "reward" -> Task.TYPE_REWARD
            else -> ""
        }

( ) , . .

response.data, , .

. onCreate TaskFormActivity.

// Inserted code for voice activation
        textEditText.setText(bundle.getString("activity_name")) // presetting task name
        notesEditText.setText(bundle.getString("activity_description")) //presetting task description
        if (bundle.getBoolean("sentiment")) {  // presetting task sentiment
            habitScoringButtons.isPositive = true
            habitScoringButtons.isNegative = false
        } else {
            habitScoringButtons.isNegative = true
            habitScoringButtons.isPositive = false
        }
        when (bundle.getString("activity_difficulty").toString()) { // presetting task difficulty
            "trivial" -> taskDifficultyButtons.selectedDifficulty = 0.1f
            "easy" -> taskDifficultyButtons.selectedDifficulty = 1f
            "medium" -> taskDifficultyButtons.selectedDifficulty = 1.5f
            "hard" -> taskDifficultyButtons.selectedDifficulty = 2f
            else -> taskDifficultyButtons.selectedDifficulty = 1f
        }

JAICF Caila.

Caila: , ( , Pattern ).

data , – habit, pattern .

, Name Description, , . .

:

, task_type . , – , , , .

, (, ). , .

, . 

@ – , “ – .

JAICF.

state("createTask") {
            activators {
                intent("createTask")
            }
            action {
                val taskType = activator.getCailaSlot("taskType").asJsonLiteralOr("")
                reactions.say("...")
                reactions.aimybox?.response?.action = "createTask"
                reactions.aimybox?.response?.intent = taskType.content
                reactions.aimybox?.response?.run {
                    data["taskName"] = activator.getCailaSlot("taskName").asJsonLiteralOr("")
                    data["taskDescription"] = activator.getCailaSlot("taskDescription").asJsonLiteralOr("")
                    data["taskSentiment"] = activator.getCailaSlotBool("taskSentiment").asJsonLiteralOr(true)
                    data["taskDifficulty"] = activator.getCailaSlot("taskDifficulty").asJsonLiteralOr("easy")
                }
            }
        } 
private fun ActivatorContext.getCailaRequiredSlot(k: String): String =
    getCailaSlot(k) ?: error("Missing Caila slot for key: $k")

private fun ActivatorContext.getCailaSlot(k: String): String? =
    caila?.slots?.get(k)

private fun ActivatorContext.getCailaSlotBool(k: String): Boolean? =
    caila?.slots?.get(k)?.toBoolean()

private fun String?.asJsonLiteralOr(other: String) = this?.let { JsonLiteral(this) } ?: JsonLiteral(other)
private fun Boolean?.asJsonLiteralOr(other: Boolean) = this?.let { JsonLiteral(this) } ?: JsonLiteral(other)

, intent, data, action, Aimybox , .

, ! :

, – , . !

JAICF.

Aimybox.