Android-based Video Call on the SRS Server (1) : ENABLE HTTPS on the SRS server (2) : The Android terminal pulls WebRTC streams from the SRS server To implement Android-based Web video calls based on the SRS server (3) : The Android terminal pushes WebRTC streams to the SRS server

Implementation effect

Lead library

implementation 'org. Webrtc: Google - webrtc: 1.0.32006'
Copy the code

For other versions, see

Push the flow process

createPeerConnectionFactory -> createPeerConnection(addTransceiver) -> createOffer -> setLocalDescription(OFFER) -> get remote sdp(network requset) -> setRemoteDescription(ANSWER)

Code implementation

Initialize the

// Load and initialize WebRTC, which must be called at least once before PeerConnectionFactory is created
PeerConnectionFactory.initialize(
    PeerConnectionFactory.InitializationOptions
        .builder(applicationContext).createInitializationOptions()
)

private val eglBaseContext = EglBase.create().eglBaseContext
Copy the code

createPeerConnectionFactory

private val peerConnectionFactory: PeerConnectionFactory = createPeerConnectionFactory()

private fun createPeerConnectionFactory(a): PeerConnectionFactory {
	// Do the default configuration first, there may be a pit later
    val options = PeerConnectionFactory.Options()
    val encoderFactory = DefaultVideoEncoderFactory(eglBaseContext, true.true)
    val decoderFactory = DefaultVideoDecoderFactory(eglBaseContext)
    return PeerConnectionFactory.builder()
        .setOptions(options)
        .setVideoEncoderFactory(encoderFactory)
        .setVideoDecoderFactory(decoderFactory)
        .createPeerConnectionFactory()
}
Copy the code

createPeerConnection(addTransceiver)

private fun initPeerConnection(a) {
    val createAudioSource = peerConnectionFactory.createAudioSource(createAudioConstraints())
    val audioTrack =
        peerConnectionFactory.createAudioTrack("local_audio_track", createAudioSource)

    cameraVideoCapturer = createVideoCapture(this) cameraVideoCapturer? .let { capture ->val videoSource = peerConnectionFactory.createVideoSource(capture.isScreencast)
        videoTrack =
            peerConnectionFactory.createVideoTrack("local_video_track", videoSource).apply {
            	// Display to local screen
                addSink(mBinding.svr)
            }
        surfaceTextureHelper =
            SurfaceTextureHelper.create("surface_texture_thread", eglBaseContext)
        capture.initialize(surfaceTextureHelper, this, videoSource.capturerObserver)
        // Start frame capture, width, height, frame rate.
        capture.startCapture(640.480.20)}val rtcConfig = PeerConnection.RTCConfiguration(emptyList())
    /* 
For users who wish to send multiple audio/video streams and need to stay interoperable with legacy WebRTC implementations, specify PLAN_B. 
For users who wish to send multiple audio/video streams and/or wish to use the new RtpTransceiver API, specify UNIFIED_PLAN. */
	/ / use the PeerConnection. SdpSemantics. UNIFIED_PLANrtcConfig.sdpSemantics = PeerConnection.SdpSemantics.UNIFIED_PLAN peerConnection = peerConnectionFactory.createPeerConnection( rtcConfig, PeerConnectionObserver() )? .apply {// Do not use addStream(), otherwise an error will be reportedvideoTrack? .let {// Add a video track and set it to send only
            addTransceiver(
                it,
                RtpTransceiver.RtpTransceiverInit(RtpTransceiver.RtpTransceiverDirection.SEND_ONLY)
            )
        }
        // Add audio track, set to send only
        addTransceiver(
            audioTrack,
            RtpTransceiver.RtpTransceiverInit(RtpTransceiver.RtpTransceiverDirection.SEND_ONLY)
        )
    }
}

private fun createAudioConstraints(a): MediaConstraints {
    val audioConstraints = MediaConstraints()
    // Echo cancellation
    audioConstraints.mandatory.add(
        MediaConstraints.KeyValuePair(
            "googEchoCancellation"."true"))// Automatic gain
    audioConstraints.mandatory.add(MediaConstraints.KeyValuePair("googAutoGainControl"."true"))
    // Treble filter
    audioConstraints.mandatory.add(MediaConstraints.KeyValuePair("googHighpassFilter"."true"))
    // Noise processing
    audioConstraints.mandatory.add(
        MediaConstraints.KeyValuePair(
            "googNoiseSuppression"."true"))return audioConstraints
}

private fun createVideoCapture(context: Context): CameraVideoCapturer? {
    val enumerator: CameraEnumerator = if (Camera2Enumerator.isSupported(context)) {
        Camera2Enumerator(context)
    } else {
        Camera1Enumerator()
    }
    for (name in enumerator.deviceNames) {
        if (enumerator.isFrontFacing(name)) {
            return enumerator.createCapturer(name, null)}}for (name in enumerator.deviceNames) {
        if (enumerator.isBackFacing(name)) {
            return enumerator.createCapturer(name, null)}}return null
}
Copy the code

createOffer && setLocalDescription

peerConnection.createOffer(object : SdpAdapter("createOffer") {
	override fun onCreateSuccess(description: SessionDescription?).{ description? .let {if (it.type == SessionDescription.Type.OFFER) {
				connection.setLocalDescription(SdpAdapter("setLocalDescription"), it)
				// This office DP will be used to make network requests to the SRS service
				val offerSdp = it.description
				getRemoteSdp(offerSdp)
			}
		}
	}
}, MediaConstraints())
Copy the code

get remote sdp(netword requset)

The basic configuration can be adjusted according to the actual situation

object Constant {
    /** * SRS server IP address */
    const val SRS_SERVER_IP = "192.168.2.91"

    /** * SRS service HTTP request port, default 1985 */
    const val SRS_SERVER_HTTP_PORT = "1985"

    /** * SRS service HTTPS request port, default 1990 */
    const val SRS_SERVER_HTTPS_PORT = "1990"

    const val SRS_SERVER_HTTP = "$SRS_SERVER_IP:$SRS_SERVER_HTTP_PORT"

    const val SRS_SERVER_HTTPS = "$SRS_SERVER_IP:$SRS_SERVER_HTTPS_PORT"
}
Copy the code

Request Body (application/json)

data class SrsRequestBean(
    / * * * [PeerConnection createOffer] returns the SDP * /
    @Json(name = "sdp")
    valsdp: String? ./** * Pull the WebRTC stream address */
    @Json(name = "streamurl")
    val streamUrl: String?
)
Copy the code

Response Body (application/json)

data class SrsResponseBean(
    /** * 0: successful */
    @Json(name = "code")
    val code: Int./ * * * is used to set [PeerConnection setRemoteDescription] * /
    @Json(name = "sdp") valsdp: String? .@Json(name = "server")
    valserver: String? .@Json(name = "sessionid")
    val sessionId: String?
)
Copy the code

Network address HTTP request: http://ip:port/rtc/v1/publish/ HTTPS requests: https://ip:port/rtc/v1/publish/ Method: POST

On Android P(28) devices, forbid applications to use HTTP network requests with unencrypted plaintext traffic.

Retrofit example

interface ApiService {

    @POST("/rtc/v1/publish/")
    suspend fun publish(@Body body: SrsRequestBean): SrsResponseBean
}
Copy the code

getRemoteSdp

private fun getRemoteSdp(offerSdp: String){
    / / address webrtc flow
    val webrtcUrl="webrtc://${Constant.SRS_SERVER_IP}/live/camera"
    val srsBean = SrsRequestBean(offerSdp, webrtcUrl)
    lifecycleScope.launch {
        val result = try {
            withContext(Dispatchers.IO) {
                retrofitClient.apiService.publish(srsBean)
            }
        } catch (e: Exception) {
            println("Network request error:${e.printStackTrace()}")
            toastError("Network request error:${e.printStackTrace()}")
            null} result? .let { bean ->if (bean.code == 0) {
                println("Network request successful, code:${bean.code}")
				setRemoteDescription(bean.sdp)
            } else {
                println("Network request failed, code:${bean.code}")}}}}Copy the code

Note: this step may return code:400, if not can be skipped.

The reasons may be:

1. If the pusher address is occupied, the solution can change the pusher address;

2. Print in the servercreate session : create session : add publisher : publish negotiate : no found valid H.264 payload type, cause: is the interface requestoffer sdpIn them=videoThere is noH.264Related information, namely WebRTC increateOfferIs returnedsdpThere is noH.264Relevant information; The SRS uses WebRTC and supports only H.264 video encoding.

In the second case, a solution can be found in another of my blog posts. So bring createPeerConnectionFactory will change: library WebRTCExtension:

private val peerConnectionFactory: PeerConnectionFactory = createPeerConnectionFactory()

private fun createPeerConnectionFactory(a): PeerConnectionFactory {
    val options = PeerConnectionFactory.Options()
    / / VideoEncoderFactory adjustment
	val encoderFactory = createCustomVideoEncoderFactory(eglBaseContext, true.true.object : VideoEncoderSupportedCallback {
                override fun isSupportedH264(info: MediaCodecInfo): Boolean {
                	// Add MediaCodecInfo to support H.264
                    return true}})val decoderFactory = DefaultVideoDecoderFactory(eglBaseContext)
    return PeerConnectionFactory.builder()
        .setOptions(options)
        .setVideoEncoderFactory(encoderFactory)
        .setVideoDecoderFactory(decoderFactory)
        .createPeerConnectionFactory()
}
Copy the code

setRemoteDescription

private fun setRemoteDescription(answerSdp: String){
	val remoteSdp = SessionDescription(SessionDescription.Type.ANSWER, /* Key points */answerSdp)
	Failed to set remote answer SDP: The order of m-lines in answer doesn't match order in offer. Rejecting answer.
	peerConnection.setRemoteDescription(SdpAdapter("setRemoteDescription"), remoteSdp)
}
Copy the code

Failed to set remote answer SDP: The order of m-lines in answer doesn’t match order in offer. Rejecting answer.

Take a look at my other blog post.

Check the offer SDP created on Android and the answer SDP returned from SRS:

offer sdp	answer sdp

Obviously, the problem is the first one in the blog category. We need to manually switch positions.

/** * convert AnswerSdp *@paramOfferSdp offerSdp: The SDP generated when an offer is created@paramAnswerSdp answerSdp: The NETWORK requests the SDP returned by the SRS server@returnAfter conversion AnswerSdp */
private fun convertAnswerSdp(offerSdp: String, answerSdp: String?).: String {
	if (answerSdp.isNullOrBlank()){
		return ""
    }
    val indexOfOfferVideo = offerSdp.indexOf("m=video")
    val indexOfOfferAudio = offerSdp.indexOf("m=audio")
    if (indexOfOfferVideo == -1 || indexOfOfferAudio == -1) {
        return answerSdp
    }
    val indexOfAnswerVideo = answerSdp.indexOf("m=video")
    val indexOfAnswerAudio = answerSdp.indexOf("m=audio")
    if (indexOfAnswerVideo == -1 || indexOfAnswerAudio == -1) {
        return answerSdp
    }

    val isFirstOfferVideo = indexOfOfferVideo < indexOfOfferAudio
    val isFirstAnswerVideo = indexOfAnswerVideo < indexOfAnswerAudio
    return if (isFirstOfferVideo == isFirstAnswerVideo) {
        // The sequence is consistent
        answerSdp
    } else {
        // The order needs to be reversed
        buildString {
            append(answerSdp.substring(0. indexOfAnswerVideo.coerceAtMost(indexOfAnswerAudio))) append( answerSdp.substring( indexOfAnswerVideo.coerceAtLeast(indexOfOfferVideo), answerSdp.length ) ) append( answerSdp.substring( indexOfAnswerVideo.coerceAtMost(indexOfAnswerAudio), indexOfAnswerVideo.coerceAtLeast(indexOfOfferVideo) ) ) } } }Copy the code

Modification method:

private fun setRemoteDescription(offerSdp: String, answerSdp: String){
	val remoteSdp = SessionDescription(SessionDescription.Type.ANSWER, /* Key points */convertAnswerSdp(offerSdp, answerSdp))
	peerConnection.setRemoteDescription(SdpAdapter("setRemoteDescription"), remoteSdp)
}
Copy the code

Shut down

Free up resources to avoid memory leaks

mBinding.svr.release() cameraVideoCapturer? .dispose() surfaceTextureHelper? .dispose() videoTrack? .dispose() peerConnection? .dispose() peerConnectionFactory.dispose()Copy the code

At this point, the push flow process ends. You are welcome to correct any mistakes.

Making portal