ios – Having bother integrating a ARViewContainer right into a CoreML playground

I attempted to combine a ARViewContainer which will get 3D coordinate from a 2D body right into a CoreML playground. Earlier than integrating, each of them works: the ARViewContainter take a CGRect body as enter after which give the 3D coordinate of the center level as output. The CoreML playground applied a yolo mannequin, body the recognized object. My anticipated habits after integration is we will get the 3D coordinate of the recognized object. However my preliminary end result, is that the ARViewContainer works effectively however CoreML mannequin appears did not work.

My ARViewContainer is like

struct ARViewContainer: UIViewRepresentable {

    @Binding var frameInput: CGRect? // Use Binding to set off updates

    func makeUIView(context: Context) -> ARSCNView {
        let arView = ARSCNView()
        let configuration = ARWorldTrackingConfiguration()
        configuration.planeDetection = [.horizontal, .vertical]
        arView.session.run(configuration)
        arView.delegate = context.coordinator

        let tapGesture = UITapGestureRecognizer(goal: context.coordinator, motion: #selector(Coordinator.handleTap))
        arView.addGestureRecognizer(tapGesture)

        return arView
    }

    func updateUIView(_ uiView: ARSCNView, context: Context) {
        if let body = frameInput {
            context.coordinator.handleFrameInput(body: body, in: uiView)
        }
    }

    func makeCoordinator() -> Coordinator {
        return Coordinator()
    }

    class Coordinator: NSObject, ARSCNViewDelegate {
        // Deal with single faucet
        @objc func handleTap(recognizer: UITapGestureRecognizer) {
            guard let view = recognizer.view as? ARSCNView else { return }
            let location = recognizer.location(in: view)
            performRaycast(at: [location], in: view)
            print("Set off Hand Enter")

        }

        // Deal with 2D body enter
        func handleFrameInput(body: CGRect, in view: ARSCNView) {
            let samplePoints = generateSamplePoints(in: body, sampleCount: 10)
            performRaycast(at: samplePoints, in: view)
            print("Set off Fram Enter")
        }

        // Carry out raycasting for a number of factors
        personal func performRaycast(at factors: [CGPoint], in view: ARSCNView) {
            for level in factors {
                guard let raycastQuery = view.raycastQuery(from: level, permitting: .estimatedPlane, alignment: .any),
                      let raycastResult = view.session.raycast(raycastQuery).first else {
                    print("No legitimate floor at level (level)")
                    proceed
                }

                let place = raycastResult.worldTransform.columns.3
                print("3D place: ((place.x), (place.y), (place.z))")

                // Non-obligatory: Add ARAnchor at this place
                let anchor = ARAnchor(rework: raycastResult.worldTransform)
                view.session.add(anchor: anchor)
            }
        }

        // Generate pattern factors inside a given CGRect
        personal func generateSamplePoints(in body: CGRect, sampleCount: Int) -> [CGPoint] {
            print("Set off Pattern Era")

            var factors = [CGPoint]()
            let stepX = body.width / CGFloat(sampleCount)
            let stepY = body.peak / CGFloat(sampleCount)

            for i in 0..

The CoreML playground is hooked up https://drive.google.com/file/d/1PddUgvLvtUmCVX9z0T2X0p_zbZzDvitv/view?usp=sharing

My present technique is described under:

since unique CoreML playground primarily run ObjectDetectionView (in ObjectDetectionView.swift), like

import ARKit
import SwiftUI

struct ObjectDetectionView {
    @State personal var state = ObjectDetectionViewState()
    @State personal var session = ARSession()
    personal let configuration: AROrientationTrackingConfiguration = {
        let configuration = AROrientationTrackingConfiguration()
        return configuration
    }()

    personal var imageResolution: CGSize { self.configuration.videoFormat.imageResolution }
    personal var cameraFPS: Double { Double(self.configuration.videoFormat.framesPerSecond) }

    personal func startSession() {
        self.session.run(self.configuration)
    }

    personal func stopSession() {
        self.session.pause()
    }
}

extension ObjectDetectionView: View {
    var physique: some View {
        ZStack {
            if self.state.isLoading {
                HStack(spacing: 5) {
                    ProgressView()
                    Textual content("Loading a mannequin...")
                }
            } else {
                self.realtimePreview
            }
        }
        .process {
            self.session.delegate = self.state
            attempt? await self.state.loadModel()
        }
        .onAppear {
            self.startSession()
        }
        .onDisappear {
            self.stopSession()
        }
    }

    personal var realtimePreview: some View {
        ZStack {
            ARViewContainer(session: self.session)
            OverlayView(frameData: self.state.frameData, imageResolution: self.imageResolution)
        }
        .ignoresSafeArea()
    }
}

Because it has an ARViewContainer mainly do nothing particular, I simply change it with my ARViewContainer, like following:

import ARKit
import SwiftUI

struct ObjectDetectionView {
    @State personal var frameInput: CGRect? = nil
    @State personal var state = ObjectDetectionViewState()
    @State personal var session = ARSession()
    personal let configuration: AROrientationTrackingConfiguration = {
        let configuration = AROrientationTrackingConfiguration()
        return configuration
    }()

    personal var imageResolution: CGSize { self.configuration.videoFormat.imageResolution }
    personal var cameraFPS: Double { Double(self.configuration.videoFormat.framesPerSecond) }

    personal func startSession() {
        self.session.run(self.configuration)
    }

    personal func stopSession() {
        self.session.pause()
    }
}    

extension ObjectDetectionView: View {
    var physique: some View {
        ZStack {
            if self.state.isLoading {
                HStack(spacing: 5) {
                    ProgressView()
                    Textual content("Loading a mannequin...")
                }
            } else {
                self.realtimePreview
            }
        }
        .process {
            self.session.delegate = self.state
            attempt? await self.state.loadModel()
        }
        .onAppear {
            self.startSession()
        }
        .onDisappear {
            self.stopSession()
        }
        .onChange(of: self.state.frameData) {  
            if let bbox = self.state.frameData?.detections.first?.bbox {
                self.frameInput = bbox
        }
    }
    }

    personal var realtimePreview: some View {
        ZStack {
            ARViewContainer(frameInput: $frameInput)
            OverlayView(frameData: self.state.frameData, imageResolution: self.imageResolution)
        }
        .ignoresSafeArea()
    }
}

The result’s the ARViewContainer works effectively however CoreML mannequin appears did not work, no object identification anymore.

Admire any insights on the right way to resolve it. DDL is approaching, lol 🙂