Recognise food using image

The Quick start project demonstrate recognising food items from an image, either by taking a photo using the camera or selecting an image from the gallery.

Include the Camera and Gallery usage keys

Include the NSCameraUsageDescription and NSPhotoLibraryUsageDescription keys in your app’s Info.plist file. It tells the user why the app is requesting access to the device’s camera and photo library.

Ask User for Camera Permission

• Once the Passio SDK status is isReadyForDetection, we can start capturing image or picking it from gallery. Implement following code for asking user for Camera permission. Once user grants the permission, we can configure the camera for capturing images.

import AVFoundation

func askForCapturePermission() {
    if AVCaptureDevice.authorizationStatus(for: .video) == .authorized {
        configureCamera()
    } else {
        AVCaptureDevice.requestAccess(for: .video) { granted in
            DispatchQueue.main.async {
                if granted {
                    self.configureCamera()
                } else {
                    self.statusLabel.text = "Please grant permission from Settings to use camera."
                }
            }
        }
    }
}

Take a Picture

• Above the viewDidLoad method, where you create variables you want to be accessible anywhere in the ViewController file, create the following Instance Variables

var captureSession: AVCaptureSession!
var stillImageOutput: AVCapturePhotoOutput!
var videoPreviewLayer: AVCaptureVideoPreviewLayer!

• Set up the Camera session and configure Input and Output

func configureCamera() {
    
    statusLabel.text = "Setting up camera..."
    captureSession = AVCaptureSession()
    captureSession.sessionPreset = .photo
    
    guard let backCamera = AVCaptureDevice.default(for: .video) else {
        statusLabel.text = "Unable to access back camera!"
        return
    }
    do {
        let input = try AVCaptureDeviceInput(device: backCamera)
        stillImageOutput = AVCapturePhotoOutput()
        
        if captureSession.canAddInput(input) &&
            captureSession.canAddOutput(stillImageOutput) {
            captureSession.addInput(input)
            captureSession.addOutput(stillImageOutput)
            setupLivePreview()
        }
    }
    catch let error {
        statusLabel.text = "Error Unable to initialize back camera:  \(error.localizedDescription)"
    }
}

• Configure the Live Preview and start the Session on the background thread

func setupLivePreview() {
    
    statusView.isHidden = true
    
    videoPreviewLayer = AVCaptureVideoPreviewLayer(session: captureSession)
    videoPreviewLayer.videoGravity = .resizeAspectFill
    videoPreviewLayer.connection?.videoOrientation = .portrait
    self.captureView.layer.insertSublayer(videoPreviewLayer, at: 0)
    
    DispatchQueue.global(qos: .userInitiated).async { [weak self] in
        guard let self = self else { return }
        self.captureSession.startRunning()
        DispatchQueue.main.async {
            self.videoPreviewLayer.frame = self.captureView.bounds
        }
    }
}

• On click of capture button, provide a setting and a deleget to deliver the capturedPhoto to. This delegate will be this ViewController so we also need to conform to the protocol AVCapturePhotoCaptureDelegate

@IBAction func captureButtonTapped(_ sender: UIButton) {
    captureImage()
}

func captureImage() {
    let settings = AVCapturePhotoSettings(format: [AVVideoCodecKey: AVVideoCode
    stillImageOutput.capturePhoto(with: settings, delegate: self)
}

• The AVCapturePhotoOutput will deliver the captured photo to the assigned delegate which is our current ViewController by a delegate method

func photoOutput(_ output: AVCapturePhotoOutput, didFinishProcessingPhoto photo: AVCapturePhoto, error: Error?) {
    guard let imageData = photo.fileDataRepresentation() else { return }
    guard let image = UIImage(data: imageData) else { return }
    // Use this image
}

Pick Picture from Gallery

• Ask for Photo Gallery permission:

PHPhotoLibrary.requestAuthorization() { status in
    DispatchQueue.main.async {
        if status == .authorized {
            self.presentImagePicker()
        } else {
            // Permission denied by user.
        }
    }
}

• Present the PHPickerViewController with configuration

func presentImagePicker() {
    var configuration = PHPickerConfiguration()
    configuration.selectionLimit = 1 // or any number
    configuration.filter = .images
    
    let picker = PHPickerViewController(configuration: configuration)
    picker.isModalInPresentation = true
    picker.delegate = self
    
    DispatchQueue.main.async {
        self.present(picker, animated: true)
    }
}

• Implement the delegate for getting user picked images

func picker(_ picker: PHPickerViewController, 
            didFinishPicking results: [PHPickerResult]) {
    
    picker.dismiss(animated: true) { [weak self] in
        
        if results.count < 1 { return }
        
        var selectedImages: [UIImage] = []
        let itemProviders = results.map(\.itemProvider)
        let dispatchGroup = DispatchGroup()
        
        for itemProvider in itemProviders {
            dispatchGroup.enter()
            if itemProvider.canLoadObject(ofClass: UIImage.self) {
                itemProvider.loadObject(ofClass: UIImage.self) { image , error  in
                    if let image = image as? UIImage {
                        selectedImages.append(image)
                        dispatchGroup.leave()
                    } else {
                        dispatchGroup.leave()
                    }
                }
            } else {
                dispatchGroup.leave()
            }
        }
        dispatchGroup.notify(queue: .main) { [weak self] in
            guard let self else { return }
            if selectedImages.count < 1 { return }
            // Use these images
        }
    }
}

Recognise the picture using SDK

The following function is responsible for sending a captured image to the Passio SDK for remote image recognition . It sends image to a remote server for recognition, and after receiving the response it updates the table view which will present a list of recognised food.

@IBOutlet weak var foodListTable: UITableView!
var foodInfo: [PassioAdvisorFoodInfo] = []

func fetchData() {
    // Pass the image we just picked from above steps
    guard let image = selectedImage else { return }
    updateLoader(show: true)
    PassioNutritionAI.shared.recognizeImageRemote(image: image) { passioAdvisorFoodInfo in
        DispatchQueue.main.async {
            self.updateLoader(show: false)
            self.didFetch(foods: passioAdvisorFoodInfo)
        }
    }
}

func didFetch(foods: [PassioAdvisorFoodInfo]) {
    if foods.count < 1 {
        // No food detected. Present the alert.
        return
    }
    foodInfo = foods
    foodListTable.reloadData()
}

Display the recognition results in the UI

For each food recognised, we can create a table view cell e.g. FoodListCell and pass the PassioAdvisorFoodInfo object

@IBOutlet weak var foodImageView: UIImageView!
@IBOutlet weak var foodNameLabel: UILabel!
@IBOutlet weak var foodDetailsLabel: UILabel!

func configure(food: PassioAdvisorFoodInfo) {
    
    if let foodInfo = food.foodDataInfo {
        
        foodImageView.loadIcon(id: foodInfo.iconID)
        foodNameLabel.text = foodInfo.foodName.capitalized
        
        if let nutritionPreview = foodInfo.nutritionPreview {
            let servingQuantity = nutritionPreview.servingQuantity.twoDigits
            foodDetailsLabel.text = "\(servingQuantity) \(nutritionPreview.servingUnit) | \(nutritionPreview.calories) cal"
        } else {
            foodDetailsLabel.text = ""
        }
    }
    
    else if let packagedFoodItem = food.packagedFoodItem {
        
        foodNameLabel.text = packagedFoodItem.name
        
        let servingQuantity = packagedFoodItem.amount.selectedQuantity.twoDigits
        let calories = packagedFoodItem.nutrientsReference().calories()?.value ?? 0
        foodDetailsLabel.text = "\(servingQuantity) \(packagedFoodItem.amount.selectedUnit) | \(calories.twoDigits) cal"
    }
    
    else {
        foodNameLabel.text = ""
        foodDetailsLabel.text = ""
    }