Run html with script in appinventor (Teachable machine / tensorflow.js)

Hello,
I am programming an app that needs to execute the following code:

<div id="webcam-container"></div>
<div id="label-container"></div>
<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@1.3.1/dist/tf.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/@teachablemachine/image@0.8/dist/teachablemachine-image.min.js"></script>
<script type="text/javascript">
    // More API functions here:
    // https://github.com/googlecreativelab/teachablemachine-community/tree/master/libraries/image

    // the link to your model provided by Teachable Machine export panel
    const URL = "https://teachablemachine.withgoogle.com/models/bpm4V6Kzk/";

    let model, webcam, labelContainer, maxPredictions;
    init()

    // Load the image model and setup the webcam
    async function init() {
        const modelURL = URL + "model.json";
        const metadataURL = URL + "metadata.json";

        // load the model and metadata
        // Refer to tmImage.loadFromFiles() in the API to support files from a file picker
        // or files from your local hard drive
        // Note: the pose library adds "tmImage" object to your window (window.tmImage)
        model = await tmImage.load(modelURL, metadataURL);
        maxPredictions = model.getTotalClasses();

        // Convenience function to setup a webcam
        const flip = true; // whether to flip the webcam
        webcam = new tmImage.Webcam(200, 200, flip); // width, height, flip
        await webcam.setup(); // request access to the webcam
        await webcam.play();
        window.requestAnimationFrame(loop);

        // append elements to the DOM
        document.getElementById("webcam-container").appendChild(webcam.canvas);
        labelContainer = document.getElementById("label-container");
        for (let i = 0; i < maxPredictions; i++) { // and class labels
            labelContainer.appendChild(document.createElement("div"));
        }
    }

    async function loop() {
        webcam.update(); // update the webcam frame
        await predict();
        window.requestAnimationFrame(loop);
    }

    // run the webcam image through the image model
    async function predict() {
        // predict can take in an image, video or canvas html element
        const prediction = await model.predict(webcam.canvas);
        for (let i = 0; i < maxPredictions; i++) {
            const classPrediction =
                prediction[i].className + ": " + prediction[i].probability.toFixed(2);
            labelContainer.childNodes[i].innerHTML = classPrediction;
        }
    }
</script>

I have just tried this:

But doesn't work :sleepy:
shows the html but does not execute the script

I hope you can help me,
thank you

1 Like

the script does't have the ability to request webcam permissions to the Android system, so it doesn't run

2 Likes

But I can apply before (with appinventor) and it will work?

Try It,but i don't think.

2 Likes

There is any other way to do this?

Possibly...

  1. Take a Picture with the device camera
  2. Base64 encode the image
  3. Pass the base64 String to the html via the webviewstring
  4. Do your teachable thing
  5. return any output via the webviewstring to the app
2 Likes

Hello Meulencv

You do need to make a complete HTML page with Header and Body. You can then pass an image to your script as described by TimAI2.

To trigger the Script, send the data via webviewstring then load or reload the HTML page and trigger the script thus:

<body onload="init()">

2 Likes

You may also want to resize (reduce dimensions) of the image before encoding to base64 in order to reduce file size / upload times / etc.

2 Likes

Thanks for all the info :grin:

It's a good idea,but i'm not sure that It works

Take a look at this

1 Like

You should be able to load the image into the div webcam-container ?

1 Like

Hello, someone could make me an example of how the html and js would look like? thanks, I can't see it.

Follow the first 5 steps in this codelab (use your computer browser). You should see that you can load an image and get the response. You can set the src for the img tag using a datauri with a base64String of an image.

1 Like

Had a play with this today, and got it working. Used Taifun's Image extension to resize, and Juan Antonio's Base64 extension for base64 conversion. Credits to both :wink: Tested with companion (2.62u) on Android 12 real device.

Tested on a few office objects and got some interesting returns (image of smartphone returned meatcleaver ....), but it is working...

tmWVS.aia (57.0 KB)

HTML
<!DOCTYPE html>
<html>
  <head>
    <!-- Load the latest version of TensorFlow.js -->
    <script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs"></script>
    <script src="https://cdn.jsdelivr.net/npm/@tensorflow-models/mobilenet"></script>
  </head>

  <body>
    <!-- Add an image that we will use to test -->
    <img id="img" src=""/>
    
    <script>
    var wvs = window.AppInventor.getWebViewString();
    document.getElementById("img").src = "data:image/jpeg;base64," + wvs;
    </script>

    <!-- Load tm script after the content of the page -->
    <script>
    
    setTimeout(() => {
    
    let net;
	async function app() {
	// Load the model.
	net = await mobilenet.load();
	// Make a prediction through the model on our image.
	const imgEl = document.getElementById('img');
	const result = await net.classify(imgEl);
	// return result via webviewstring
	window.AppInventor.setWebViewString(JSON.stringify(result));	
	}
	app();
	
	}, 300);
	
    </script>
  </body>
</html>
3 Likes

Really? It appears to be a random 'number' guesser. MIT's PIC certainly does a better job. So far this results in 0 for six objects and the guesses are not even close. :cry: Score: TM 0 for 6; PIC 5 for 6. :astonished:

Have fun. :slight_smile:

1 Like

the html is working, I say nothing about the image classifier :wink:

1 Like

But, with this you are not using a custom project of teachable machine, I want to use with for example this: Teachable Machine

I have managed to create this html code with this and this
The problem is that it does not give me the prediction and I do not know why, someone who knows html help me please? thanks

My code Teachable Machine Image Model with upload
	<div>Teachable Machine Image Model with upload</div>
	<div id="label-container"></div>
	<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@1.3.1/dist/tf.min.js"></script>
	<script src="https://cdn.jsdelivr.net/npm/@teachablemachine/image@0.8/dist/teachablemachine-image.min.js"></script>
	<script type="text/javascript">
		// More API functions here:
		// https://github.com/googlecreativelab/teachablemachine-community/tree/master/libraries/image

		// the link to your model provided by Teachable Machine export panel
		const URL = 'https://teachablemachine.withgoogle.com/models/bpm4V6Kzk/';

		let model, labelContainer, maxPredictions;

		// Load the image model 
		async function init() {
			const modelURL = URL + 'model.json';
			const metadataURL = URL + 'metadata.json';

			// load the model and metadata
			model = await tmImage.load(modelURL, metadataURL);
			maxPredictions = model.getTotalClasses();

			labelContainer = document.getElementById('label-container');
			for (let i = 0; i < maxPredictions; i++) {
				// and class labels
				labelContainer.appendChild(document.createElement('div'));
			}
		}

		async function predict() {
			// predict can take in an image, video or canvas html element
			var image = document.getElementById('imagePreview');
			const prediction = await model.predict(image, false);
			for (let i = 0; i < maxPredictions; i++) {
				const classPrediction =
					prediction[i].className + ': ' + prediction[i].probability.toFixed(2);
				labelContainer.childNodes[i].innerHTML = classPrediction;
			}
		}
	</script>

	<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.1.1/jquery.min.js"></script>
	<script type="text/javascript">
var wvs = window.AppInventor.getWebViewString();
document.getElementById("imagePreview").src = "data:image/jpeg;base64," + wvs;

				init().then(() => {
			    predict();
				});
        

	</script>

</body>

And what leads you to that conclusion. I followed the guide as linked above, and the teachable machine returns results....

1 Like

Sorry....
In one of my tests it did not work.
Now it works, you are using taechable machine, the problem is that I want to use it with a project created by me https://teachablemachine.withgoogle.com/models/bpm4V6Kzk/
sorry for my misunderstanding about your code.