diff --git a/daemon/call.c b/daemon/call.c index 25adfb8d7..f5044ec11 100644 --- a/daemon/call.c +++ b/daemon/call.c @@ -69,7 +69,6 @@ static int64_t add_ongoing_calls_dur_in_interval(int64_t interval_start, int64_t static void __call_free(call_t *p); static void __call_cleanup(call_t *c); static void __monologue_stop(struct call_monologue *ml); -static void media_stop(struct call_media *m); __attribute__((nonnull(1, 2, 4))) static struct media_subscription *__subscribe_medias_both_ways(struct call_media * a, struct call_media * b, bool is_offer, medias_q *); @@ -5182,7 +5181,7 @@ static void __call_cleanup(call_t *c) { for (__auto_type l = c->medias.head; l; l = l->next) { struct call_media *md = l->data; ice_shutdown(&md->ice_agent); - media_stop(md); + call_media_stop(md); t38_gateway_put(&md->t38_gateway); audio_player_free(md); mutex_destroy(&md->dtmf_lock); @@ -5498,6 +5497,7 @@ static void __call_free(call_t *c) { //ilog(LOG_DEBUG, "freeing main call struct"); + call_checkpoint_free_all(c); obj_release(c->dtls_cert); mqtt_timer_stop(&c->mqtt_timer); @@ -5748,6 +5748,9 @@ static bool call_merge(call_t *call, call_t *call2) { // move buffers bencode_buffer_merge(&call->buffer, &call2->buffer); + // the ids below are about to be renumbered, and a snapshot is keyed on them + call_checkpoint_free_all(call2); + // move all contained objects: we have to renumber all unique IDs, and redirect any // `call` pointers @@ -6418,7 +6421,7 @@ int call_get_mono_dialogue(struct call_monologue *monologues[2], return call_get_dialogue(monologues, call, callid, fromtag, totag, viabranch, flags, ep); } -static void media_stop(struct call_media *m) { +void call_media_stop(struct call_media *m) { if (!m) return; t38_gateway_stop(m->t38_gateway); @@ -6443,7 +6446,7 @@ static void monologue_stop(struct call_monologue *ml, bool stop_media_subscriber __monologue_stop(ml); for (unsigned int i = 0; i < ml->medias->len; i++) { - media_stop(ml->medias->pdata[i]); + call_media_stop(ml->medias->pdata[i]); } /* monologue's subscribers */ if (stop_media_subscribers) { @@ -6453,7 +6456,7 @@ static void monologue_stop(struct call_monologue *ml, bool stop_media_subscriber if (!media) continue; IQUEUE_FOREACH(&media->media_subscribers, ms) { - media_stop(ms->media); + call_media_stop(ms->media); __monologue_stop(ms->monologue); } } @@ -6869,3 +6872,66 @@ void call_q_unlock_release(call_q *calls) { call_unlock_release(call); } } + + +static void checkpoint_clear_snapshot(struct call_checkpoint *cp) { + redis_snapshot_free(&cp->snapshot); + cp->pending = false; +} + +static void checkpoint_offer_one(call_t *call, struct call_monologue *ml, bool enable) { + if (!ml) + return; + if (!ml->checkpoint) { + if (!enable) + return; + ml->checkpoint = g_new0(__typeof(*ml->checkpoint), 1); + } + // consecutive offers belong to the same uncommitted exchange: keep the + // committed snapshot, or a later rollback restores a rejected offer + if (ml->checkpoint->pending) + return; + checkpoint_clear_snapshot(ml->checkpoint); + ml->checkpoint->snapshot = redis_snapshot_encode(call, ml); + ml->checkpoint->pending = true; +} + +void call_checkpoint_offer(call_t *call, struct call_monologue *offerer, + struct call_monologue *answerer, bool enable) +{ + // a dialogue is tracked if either side is, so both are checkpointed together + bool tracked = enable + || (offerer && offerer->checkpoint) + || (answerer && answerer->checkpoint); + checkpoint_offer_one(call, offerer, tracked); + checkpoint_offer_one(call, answerer, tracked); +} + +static void checkpoint_commit_one(struct call_monologue *ml) { + if (ml && ml->checkpoint && ml->checkpoint->pending) + checkpoint_clear_snapshot(ml->checkpoint); +} + +void call_checkpoint_answer(call_t *call, struct call_monologue *a, struct call_monologue *b) { + checkpoint_commit_one(a); + checkpoint_commit_one(b); +} + +int call_checkpoint_rollback(call_t *call, struct call_monologue *a, struct call_monologue *b) { + if (!redis_snapshot_apply(call, a, b)) + return 0; + + call->last_signal_us = rtpe_now; + return 1; +} + +void call_checkpoint_free_all(call_t *call) { + for (__auto_type l = call->monologues.head; l; l = l->next) { + struct call_monologue *ml = l->data; + if (!ml->checkpoint) + continue; + redis_snapshot_free(&ml->checkpoint->snapshot); + g_free(ml->checkpoint); + ml->checkpoint = NULL; + } +} diff --git a/daemon/call_flags.c b/daemon/call_flags.c index 6b6cdd4dd..5ba708638 100644 --- a/daemon/call_flags.c +++ b/daemon/call_flags.c @@ -526,6 +526,8 @@ static const char *call_ng_flags_supports(str *s, unsigned int idx, helper_arg a sdp_ng_flags *out = arg.flags; if (!str_cmp(s, "load limit")) out->supports_load_limit = true; + else if (!str_cmp(s, "rollback")) + out->supports_rollback = true; else ilog(LOG_INFO | LOG_FLAG_LIMIT, "Optional feature '" STR_FORMAT "' not supported", STR_FMT(s)); @@ -909,6 +911,10 @@ const char *call_ng_flags_flags(str *s, unsigned int idx, helper_arg arg) { case CSH_LOOKUP("reset"): out->reset = true; break; + case CSH_LOOKUP("track-state"): + case CSH_LOOKUP("track state"): + out->track_state = true; + break; case CSH_LOOKUP("single-codec"): case CSH_LOOKUP("single codec"): out->single_codec = true; diff --git a/daemon/call_interfaces.c b/daemon/call_interfaces.c index ab004a095..c64e5a402 100644 --- a/daemon/call_interfaces.c +++ b/daemon/call_interfaces.c @@ -652,10 +652,18 @@ static const char *call_offer_answer_ng(ng_command_ctx_t *ctx, const char *addr) t_hash_table_insert(call->endpoints, memory_arena_objdup(streams.head->data->rtp_endpoint), from_ml); + if (flags.opmode == OP_OFFER) { + // opened before SDP processing, to capture the pre-offer state. left + // pending if the offer is rejected: it's still the committed state + call_checkpoint_offer(call, from_ml, to_ml, flags.track_state); + } + struct recording *recording = NULL; /* offer/answer model processing */ if ((ret = monologue_offer_answer(monologues, &streams, &flags)) == 0) { + if (flags.opmode == OP_ANSWER) + call_checkpoint_answer(call, from_ml, to_ml); update_metadata_monologue(from_ml, &flags); detect_setup_recording(call, &flags); @@ -681,6 +689,10 @@ static const char *call_offer_answer_ng(ng_command_ctx_t *ctx, const char *addr) /* place return output SDP */ ctx->ngbuf->sdp_out = sdp_out.s; ctx->parser_ctx.parser->dict_add_str(output, "sdp", &sdp_out); + if (flags.supports_rollback) { + parser_arg supported = parser->dict_add_list(output, "supported"); + parser->list_add_string(supported, "rollback"); + } meta_write_sdp_after(recording, &sdp_out, from_ml, flags.opmode); @@ -734,6 +746,59 @@ const char *call_answer_ng(ng_command_ctx_t *ctx) { return call_offer_answer_ng(ctx, NULL); } +static bool monologue_has_tag(const struct call_monologue *ml, const str *tag) { + if (!str_cmp_str(&ml->tag, tag)) + return true; + for (__auto_type l = ml->tag_aliases.head; l; l = l->next) { + if (!str_cmp_str(l->data, tag)) + return true; + } + return false; +} + +const char *call_rollback_ng(ng_command_ctx_t *ctx) { + const ng_parser_t *parser = ctx->parser_ctx.parser; + str call_id = parser->dict_get_str(ctx->req, "call-id"); + str from_tag = parser->dict_get_str(ctx->req, "from-tag"); + str to_tag = parser->dict_get_str(ctx->req, "to-tag"); + str via_branch = parser->dict_get_str(ctx->req, "via-branch"); + + if (!call_id.len) + return "No call-id in message"; + if (!from_tag.len) + return "No from-tag in message"; + if (!to_tag.len) + return "No to-tag in message"; + + call_t *call = call_get(&call_id); + if (!call) + return "Unknown call-id"; + + struct call_monologue *from_ml = call_get_monologue(call, &from_tag); + struct call_monologue *to_ml = via_branch.len + ? t_hash_table_lookup(call->viabranches, &via_branch) + : call_get_monologue(call, &to_tag); + // call_get_monologue() is keyed on the tag, so from_ml carries it by + // construction. to_ml may have come from the viabranch table instead. + if (!from_ml || !to_ml || from_ml == to_ml + || !monologue_has_tag(to_ml, &to_tag) + || !g_hash_table_contains(from_ml->associated_tags, to_ml)) + { + rwlock_unlock_w(&call->master_lock); + obj_put(call); + return "Unknown dialogue"; + } + if (rtpe_config.active_switchover && IS_FOREIGN_CALL(call)) + call_make_own_foreign(call, false); + + int rolled_back = call_checkpoint_rollback(call, from_ml, to_ml); + parser->dict_add_int(ctx->resp, "rolled-back", rolled_back); + rwlock_unlock_w(&call->master_lock); + redis_update_onekey(call, rtpe_redis_write); + obj_put(call); + return NULL; +} + const char *call_delete_ng(ng_command_ctx_t *ctx) { g_auto(sdp_ng_flags) rtpp_flags; parser_arg output = ctx->resp; diff --git a/daemon/ice.c b/daemon/ice.c index dde134897..51e4f141c 100644 --- a/daemon/ice.c +++ b/daemon/ice.c @@ -53,6 +53,7 @@ static void __agent_schedule(struct ice_agent *ag, int64_t); static void __agent_schedule_abs(struct ice_agent *ag, int64_t tv); static void __agent_deschedule(struct ice_agent *ag); static void __ice_agent_free_components(struct ice_agent *ag); +static void __ice_pairings(struct ice_agent *ag); static void __agent_shutdown(struct ice_agent *ag); static void ice_agents_timer_run(void *); @@ -358,7 +359,7 @@ TYPED_GHASHTABLE_IMPL(foundation_ht, __found_hash, __found_equal, NULL, NULL) TYPED_GHASHTABLE_IMPL(priority_ht, g_direct_hash, g_direct_equal, NULL, NULL) TYPED_GHASHTABLE_IMPL(transaction_ht, __trans_hash, __trans_equal, NULL, NULL) -static void __ice_agent_initialize(struct ice_agent *ag) { +static void __ice_agent_initialize(struct ice_agent *ag, bool generate_credentials) { struct call_media *media = ag->media; call_t *call = ag->call; @@ -377,8 +378,10 @@ static void __ice_agent_initialize(struct ice_agent *ag) { ag->succeeded_pairs = g_tree_new(__pair_prio_cmp); ag->all_pairs = g_tree_new(__pair_prio_cmp); - create_random_ice_string(call, &ag->ufrag[1], 8); - create_random_ice_string(call, &ag->pwd[1], 26); + if (generate_credentials) { + create_random_ice_string(call, &ag->ufrag[1], 8); + create_random_ice_string(call, &ag->pwd[1], 26); + } atomic64_set_na(&ag->last_activity, rtpe_now); } @@ -394,7 +397,7 @@ static struct ice_agent *__ice_agent_new(struct call_media *media) { ag->media = media; mutex_init(&ag->lock); - __ice_agent_initialize(ag); + __ice_agent_initialize(ag, true); return ag; } @@ -416,14 +419,40 @@ static unsigned int __copy_cand(call_t *call, struct ice_candidate *dst, const s return eq ? 0 : 1; } -static void __ice_reset(struct ice_agent *ag) { +// generate_credentials = new local credentials; not wanted for a rollback, +// which installs the ones the committed exchange agreed +static void __ice_reset(struct ice_agent *ag, bool generate_credentials) { __agent_deschedule(ag); AGENT_CLEAR3(ag, COMPLETED, NOMINATING, USABLE); __ice_agent_free_components(ag); ZERO(ag->active_components); ag->start_nominating = 0; ag->tt_obj.last_run = 0; - __ice_agent_initialize(ag); + __ice_agent_initialize(ag, generate_credentials); +} + +/* called with the call lock held in W, hence agent doesn't need to be locked */ +void ice_rollback(struct ice_agent *ag, const str ufrag[2], const str pwd[2], + const candidate_q *candidates) +{ + if (!ag) + return; + + __ice_reset(ag, false); + memcpy(ag->ufrag, ufrag, sizeof(ag->ufrag)); + memcpy(ag->pwd, pwd, sizeof(ag->pwd)); + + for (__auto_type l = candidates->head; l; l = l->next) { + struct ice_candidate *copy = g_new(__typeof(*copy), 1); + *copy = *(struct ice_candidate *) l->data; + t_hash_table_insert(ag->candidate_hash, copy, copy); + t_hash_table_insert(ag->cand_prio_hash, GUINT_TO_POINTER(copy->priority), copy); + t_hash_table_insert(ag->foundation_hash, copy, copy); + t_queue_push_tail(&ag->remote_candidates, copy); + ag->active_components = MAX(ag->active_components, copy->component_id); + } + __ice_pairings(ag); + ice_start(ag); } /* if the other side did a restart */ @@ -434,7 +463,7 @@ static void __ice_restart(struct ice_agent *ag) { ag->pwd[0] = STR_NULL; ag->ufrag[1] = STR_NULL; ag->pwd[1] = STR_NULL; - __ice_reset(ag); + __ice_reset(ag, true); } /* if we're doing a restart */ @@ -443,7 +472,7 @@ void ice_restart(struct ice_agent *ag) { ag->ufrag[1] = STR_NULL; ag->pwd[1] = STR_NULL; - __ice_reset(ag); + __ice_reset(ag, true); } /* called with the call lock held in W, hence agent doesn't need to be locked */ diff --git a/daemon/redis.c b/daemon/redis.c index c4547afe4..3141361a8 100644 --- a/daemon/redis.c +++ b/daemon/redis.c @@ -21,6 +21,7 @@ #include "compat.h" #include "helpers.h" #include "call.h" +#include "ice.h" #include "log_d.h" #include "str.h" #include "crypto.h" @@ -1080,6 +1081,14 @@ static const char *json_get_hash_iter(const ng_parser_t *parser, str *key, parse return NULL; } +int redis_hash_from_parser(struct redis_hash *out, const ng_parser_t *parser, parser_arg dict) { + out->ht = g_hash_table_new_full(g_str_hash, g_str_equal, free, free); + if (!out->ht) + return -1; + parser->dict_iter(parser, dict, json_get_hash_iter, out->ht); + return 0; +} + static int json_get_hash(struct redis_hash *out, const char *key, unsigned int id, parser_arg root) { @@ -1103,16 +1112,10 @@ static int json_get_hash(struct redis_hash *out, return -1; } - out->ht = g_hash_table_new_full(g_str_hash, g_str_equal, free, free); - if (!out->ht) - return -1; - - redis_parser->dict_iter(redis_parser, dict, json_get_hash_iter, out->ht); - - return 0; + return redis_hash_from_parser(out, redis_parser, dict); } -static void json_destroy_hash(struct redis_hash *rh) { +void redis_hash_destroy(struct redis_hash *rh) { g_hash_table_destroy(rh->ht); } @@ -1120,7 +1123,7 @@ static void json_destroy_list(struct redis_list *rl) { unsigned int i; for (i = 0; i < rl->len; i++) { - json_destroy_hash(&rl->rh[i]); + redis_hash_destroy(&rl->rh[i]); } free(rl->rh); free(rl->ptrs); @@ -1256,6 +1259,8 @@ static int redis_hash_get_endpoint(struct endpoint *out, const struct redis_hash return 0; } +define_get_type_format(endpoint, struct endpoint); + static int redis_hash_get_stats(struct stream_stats *out, const struct redis_hash *h, const char *k) { if (redis_hash_get_a64_f(&out->packets, h, "%s-packets", k)) return -1; @@ -1369,7 +1374,7 @@ static int json_get_list_hash(struct redis_list *out, free(out->ptrs); while (i) { i--; - json_destroy_hash(&out->rh[i]); + redis_hash_destroy(&out->rh[i]); } err1: free(out->rh); @@ -1418,7 +1423,7 @@ static int redis_hash_get_sdes_params1(struct crypto_params *out, const struct r rlog(LOG_ERR, "Crypto params error: %s", err); return -1; } -static int redis_hash_get_sdes_params(sdes_q *out, const struct redis_hash *h, const char *k) { +int redis_decode_sdes_params(sdes_q *out, const struct redis_hash *h, const char *k) { char key[32], tagkey[64]; const char *kk = k; unsigned int tag; @@ -1445,6 +1450,17 @@ static int redis_hash_get_sdes_params(sdes_q *out, const struct redis_hash *h, c return 0; } +int redis_decode_dtls_fingerprint(struct dtls_fingerprint *out, const struct redis_hash *h) { + str hash; + if (redis_hash_get_str(&hash, h, "hash_func")) + return 0; + out->hash_func = dtls_find_hash_func(&hash); + if (!out->hash_func || redis_hash_get_c_buf_f(out->digest, h, "fingerprint")) + return -1; + out->digest_len = out->hash_func->num_bytes; + return 0; +} + static int redis_sfds(call_t *c, struct redis_list *sfds) { unsigned int i; str family, intf_name; @@ -1514,6 +1530,16 @@ static int redis_sfds(call_t *c, struct redis_list *sfds) { return -1; } +static int redis_decode_stream_fields(struct packet_stream *ps, const struct redis_hash *rh) { + if (redis_hash_get_a64(&ps->ps_flags, rh, "ps_flags")) + return -1; + if (redis_hash_get_endpoint(&ps->endpoint, rh, "endpoint")) + return -1; + if (redis_hash_get_endpoint(&ps->advertised_endpoint, rh, "advertised_endpoint")) + return -1; + return 0; +} + static int redis_streams(call_t *c, struct redis_list *streams) { unsigned int i; struct redis_hash *rh; @@ -1527,14 +1553,10 @@ static int redis_streams(call_t *c, struct redis_list *streams) { return -1; atomic64_set_na(&ps->last_packet_us, now_us()); - if (redis_hash_get_a64(&ps->ps_flags, rh, "ps_flags")) + if (redis_decode_stream_fields(ps, rh)) return -1; if (redis_hash_get_unsigned((unsigned int *) &ps->component, rh, "component")) return -1; - if (redis_hash_get_endpoint(&ps->endpoint, rh, "endpoint")) - return -1; - if (redis_hash_get_endpoint(&ps->advertised_endpoint, rh, "advertised_endpoint")) - return -1; if (redis_hash_get_stats(ps->stats_in, rh, "stats")) return -1; if (redis_hash_get_sdes_params1(&ps->crypto.params, rh, "") == -1) @@ -1545,10 +1567,56 @@ static int redis_streams(call_t *c, struct redis_list *streams) { return 0; } +static void redis_decode_monologue_sdp(struct call_monologue *ml, const struct redis_hash *rh) { + str s; + long il; + /* s= */ + if (!redis_hash_get_str(&s, rh, "sdp_session_name")) + ml->sdp_session_name = call_str_cpy(&s); + /* t= */ + if (!redis_hash_get_str(&s, rh, "sdp_session_timing")) + ml->sdp_session_timing = call_str_cpy(&s); + /* o= */ + if (!redis_hash_get_str(&s, rh, "sdp_orig_parsed")) { + ml->sdp_orig_in.parsed = 1; + redis_hash_get_llu(&ml->sdp_orig_in.version_num, rh, "sdp_orig_version_num"); + if (!redis_hash_get_str(&s, rh, "sdp_orig_username")) + ml->sdp_orig_in.username = call_str_cpy(&s); + if (!redis_hash_get_str(&s, rh, "sdp_orig_session_id")) + ml->sdp_orig_in.session_id = call_str_cpy(&s); + if (!redis_hash_get_str(&s, rh, "sdp_orig_address_network_type")) + ml->sdp_orig_in.address.network_type = call_str_cpy(&s); + if (!redis_hash_get_str(&s, rh, "sdp_orig_address_address_type")) + ml->sdp_orig_in.address.address_type = call_str_cpy(&s); + if (!redis_hash_get_str(&s, rh, "sdp_orig_address_address")) + ml->sdp_orig_in.address.address = call_str_cpy(&s); + } + /* o= last used of the other side*/ + if (!redis_hash_get_str(&s, rh, "last_sdp_orig_parsed")) { + ml->sdp_orig_out.parsed = 1; + redis_hash_get_llu(&ml->sdp_orig_out.version_num, rh, "last_sdp_orig_version_num"); + if (!redis_hash_get_str(&s, rh, "last_sdp_orig_username")) + ml->sdp_orig_out.username = call_str_cpy(&s); + if (!redis_hash_get_str(&s, rh, "last_sdp_orig_session_id")) + ml->sdp_orig_out.session_id = call_str_cpy(&s); + if (!redis_hash_get_str(&s, rh, "last_sdp_orig_address_network_type")) + ml->sdp_orig_out.address.network_type = call_str_cpy(&s); + if (!redis_hash_get_str(&s, rh, "last_sdp_orig_address_address_type")) + ml->sdp_orig_out.address.address_type = call_str_cpy(&s); + if (!redis_hash_get_str(&s, rh, "last_sdp_orig_address_address")) + ml->sdp_orig_out.address.address = call_str_cpy(&s); + } + + ml->sdp_session_bandwidth.as = (!redis_hash_get_ld(&il, rh, "sdp_session_as")) ? il : -1; + ml->sdp_session_bandwidth.ct = (!redis_hash_get_ld(&il, rh, "sdp_session_ct")) ? il : -1; + ml->sdp_session_bandwidth.rr = (!redis_hash_get_ld(&il, rh, "sdp_session_rr")) ? il : -1; + ml->sdp_session_bandwidth.rs = (!redis_hash_get_ld(&il, rh, "sdp_session_rs")) ? il : -1; + ml->sdp_session_bandwidth.tias = (!redis_hash_get_ld(&il, rh, "sdp_session_tias")) ? il : -1; +} + static int redis_tags(call_t *c, struct redis_list *tags, parser_arg arg) { unsigned int i; int ii; - long il; atomic64 a64; struct redis_hash *rh; struct call_monologue *ml; @@ -1579,47 +1647,7 @@ static int redis_tags(call_t *c, struct redis_list *tags, parser_arg arg) { if (!redis_hash_get_a64(&a64, rh, "ml_flags")) ml->ml_flags = a64; - /* s= */ - if (!redis_hash_get_str(&s, rh, "sdp_session_name")) - ml->sdp_session_name = call_str_cpy(&s); - /* t= */ - if (!redis_hash_get_str(&s, rh, "sdp_session_timing")) - ml->sdp_session_timing = call_str_cpy(&s); - /* o= */ - if (!redis_hash_get_str(&s, rh, "sdp_orig_parsed")) { - ml->sdp_orig_in.parsed = 1; - redis_hash_get_llu(&ml->sdp_orig_in.version_num, rh, "sdp_orig_version_num"); - if (!redis_hash_get_str(&s, rh, "sdp_orig_username")) - ml->sdp_orig_in.username = call_str_cpy(&s); - if (!redis_hash_get_str(&s, rh, "sdp_orig_session_id")) - ml->sdp_orig_in.session_id = call_str_cpy(&s); - if (!redis_hash_get_str(&s, rh, "sdp_orig_address_network_type")) - ml->sdp_orig_in.address.network_type = call_str_cpy(&s); - if (!redis_hash_get_str(&s, rh, "sdp_orig_address_address_type")) - ml->sdp_orig_in.address.address_type = call_str_cpy(&s); - if (!redis_hash_get_str(&s, rh, "sdp_orig_address_address")) - ml->sdp_orig_in.address.address = call_str_cpy(&s); - } - /* o= last used of the other side*/ - if (!redis_hash_get_str(&s, rh, "last_sdp_orig_parsed")) { - ml->sdp_orig_out.parsed = 1; - redis_hash_get_llu(&ml->sdp_orig_out.version_num, rh, "last_sdp_orig_version_num"); - if (!redis_hash_get_str(&s, rh, "last_sdp_orig_username")) - ml->sdp_orig_out.username = call_str_cpy(&s); - if (!redis_hash_get_str(&s, rh, "last_sdp_orig_session_id")) - ml->sdp_orig_out.session_id = call_str_cpy(&s); - if (!redis_hash_get_str(&s, rh, "last_sdp_orig_address_network_type")) - ml->sdp_orig_out.address.network_type = call_str_cpy(&s); - if (!redis_hash_get_str(&s, rh, "last_sdp_orig_address_address_type")) - ml->sdp_orig_out.address.address_type = call_str_cpy(&s); - if (!redis_hash_get_str(&s, rh, "last_sdp_orig_address_address")) - ml->sdp_orig_out.address.address = call_str_cpy(&s); - } - - ml->sdp_session_bandwidth.as = (!redis_hash_get_ld(&il, rh, "sdp_session_as")) ? il : -1; - ml->sdp_session_bandwidth.ct = (!redis_hash_get_ld(&il, rh, "sdp_session_ct")) ? il : -1; - ml->sdp_session_bandwidth.rr = (!redis_hash_get_ld(&il, rh, "sdp_session_rr")) ? il : -1; - ml->sdp_session_bandwidth.rs = (!redis_hash_get_ld(&il, rh, "sdp_session_rs")) ? il : -1; + redis_decode_monologue_sdp(ml, rh); if (redis_hash_get_str(&s, rh, "desired_family")) return -1; @@ -1653,16 +1681,54 @@ static rtp_payload_type *rbl_cb_plts_g(str *s, struct redis_list *list, void *pt return pt; } -static int rbl_cb_plts_r(str *s, callback_arg_t dummy, struct redis_list *list, void *ptr) { - struct call_media *med = ptr; - codec_store_add_raw(&med->codecs, rbl_cb_plts_g(s, list, ptr)); +static const char *redis_decode_codec_iter(str *value, unsigned int i, helper_arg arg) { + struct codec_store *store = arg.generic; + str *decoded = redis_parser->unescape(value->s, value->len); + struct call_media *media = store->media; + rtp_payload_type *pt = rbl_cb_plts_g(decoded, NULL, media); + g_free(decoded); + if (!pt) + return "invalid payload type"; + codec_store_add_raw(store, pt); + return NULL; +} + +int redis_decode_codec_store(const ng_parser_t *parser, parser_arg list, struct codec_store *store) { + const ng_parser_t *saved = redis_parser; + redis_parser = parser; + const char *err = parser->list_iter(parser, list, redis_decode_codec_iter, NULL, store); + redis_parser = saved; + return err ? -1 : 0; +} + +static int redis_decode_media_fields(struct call_media *med, const struct redis_hash *rh) { + str s; + long il; + + if (redis_hash_get_int(&med->ptime, rh, "ptime")) + return -1; + if (redis_hash_get_int(&med->maxptime, rh, "maxptime")) + return -1; + if (redis_hash_get_str(&s, rh, "protocol")) + return -1; + med->protocol = transport_protocol(&s); + if (redis_hash_get_str(&s, rh, "desired_family")) + return -1; + med->desired_family = get_socket_family_rfc(&s); + + med->format_str = !redis_hash_get_str(&s, rh, "format_str") ? call_str_cpy(&s) : STR_NULL; + + /* bandwidth data is not critical */ + med->sdp_media_bandwidth.as = (!redis_hash_get_ld(&il, rh, "bandwidth_as")) ? il : -1; + med->sdp_media_bandwidth.rr = (!redis_hash_get_ld(&il, rh, "bandwidth_rr")) ? il : -1; + med->sdp_media_bandwidth.rs = (!redis_hash_get_ld(&il, rh, "bandwidth_rs")) ? il : -1; return 0; } + static int json_medias(call_t *c, struct redis_list *medias, struct redis_list *tags, parser_arg arg) { unsigned int i; - long il; struct redis_hash *rh; struct call_media *med; str s; @@ -1679,23 +1745,11 @@ static int json_medias(call_t *c, struct redis_list *medias, struct redis_list * return -1; med->type = call_str_cpy(&s); med->type_id = codec_get_type(&med->type); - if (!redis_hash_get_str(&s, rh, "format_str")) - med->format_str = call_str_cpy(&s); if (!redis_hash_get_str(&s, rh, "media_id")) med->media_id = call_str_cpy(&s); - if (redis_hash_get_int(&med->ptime, rh, "ptime")) - return -1; - if (redis_hash_get_int(&med->maxptime, rh, "maxptime")) - return -1; - - if (redis_hash_get_str(&s, rh, "protocol")) - return -1; - med->protocol = transport_protocol(&s); - - if (redis_hash_get_str(&s, rh, "desired_family")) + if (redis_decode_media_fields(med, rh)) return -1; - med->desired_family = get_socket_family_rfc(&s); if (!redis_hash_get_str(&s, rh, "logical_intf") && !(med->logical_intf = get_logical_interface(&s, med->desired_family, 0))) @@ -1708,17 +1762,17 @@ static int json_medias(call_t *c, struct redis_list *medias, struct redis_list * "media_flags")) return -1; - if (redis_hash_get_sdes_params(&med->sdes_in, rh, "sdes_in") < 0) + if (redis_decode_sdes_params(&med->sdes_in, rh, "sdes_in") < 0) return -1; - if (redis_hash_get_sdes_params(&med->sdes_out, rh, "sdes_out") < 0) + if (redis_decode_sdes_params(&med->sdes_out, rh, "sdes_out") < 0) return -1; - /* bandwidth data is not critical */ - med->sdp_media_bandwidth.as = (!redis_hash_get_ld(&il, rh, "bandwidth_as")) ? il : -1; - med->sdp_media_bandwidth.rr = (!redis_hash_get_ld(&il, rh, "bandwidth_rr")) ? il : -1; - med->sdp_media_bandwidth.rs = (!redis_hash_get_ld(&il, rh, "bandwidth_rs")) ? il : -1; - json_build_list_cb(NULL, c, "payload_types", i, NULL, rbl_cb_plts_r, med, arg); + char payload_key[64]; + snprintf(payload_key, sizeof(payload_key), "payload_types-%u", i); + parser_arg payloads = redis_parser->dict_get_expect(arg, payload_key, BENCODE_LIST); + if (payloads.gen && redis_decode_codec_store(redis_parser, payloads, &med->codecs)) + return -1; /* XXX dtls */ /* link monologue */ @@ -1918,6 +1972,8 @@ static int json_link_streams(call_t *c, struct redis_list *streams, if (json_build_list(&ps->sfds, c, "stream_sfds", i, sfds, arg)) return -1; + for (__auto_type sfd_link = ps->sfds.head; sfd_link; sfd_link = sfd_link->next) + stream_fd_inc(sfd_link->data); if (json_build_list(&q, c, "rtp_sinks", i, streams, arg)) return -1; @@ -2034,6 +2090,11 @@ static int json_link_maps(call_t *c, struct redis_list *maps, if (json_build_list_cb(&em->intf_sfds, c, "map_sfds", em->unique_id, sfds, rbl_cb_intf_sfds, em, arg)) return -1; + for (__auto_type l = em->intf_sfds.head; l; l = l->next) { + struct sfd_intf_list *il = l->data; + for (__auto_type k = il->list.head; k; k = k->next) + stream_fd_inc(k->data); + } } return 0; } @@ -2073,6 +2134,101 @@ static int json_build_ssrc(struct call_media *md, parser_arg arg) { return 0; } +static int checkpoint_get_int(int64_t *out, const struct redis_hash *h, const char *k) { + str *s = g_hash_table_lookup(h->ht, k); + if (!s || !s->len) + return -1; + char *end = NULL; + errno = 0; + long long v = strtoll(s->s, &end, 10); + if (errno || !end || end != s->s + s->len) + return -1; + *out = v; + return 0; +} + +static int redis_restore_checkpoints(call_t *c, parser_arg root) { + for (__auto_type l = c->monologues.head; l; l = l->next) { + struct call_monologue *ml = l->data; + struct redis_hash rh; + // absent for a call written by a version that had no checkpoints + if (json_get_hash(&rh, "checkpoint", ml->unique_id, root)) + continue; + + int64_t pending = 0; + str snap = STR_NULL; + int bad = checkpoint_get_int(&pending, &rh, "pending"); + /* the hash owns its values; copy out before it's destroyed */ + if (!bad) { + str stored; + if (!redis_hash_get_str(&stored, &rh, "snapshot")) + snap = str_dup_str(&stored); + } + redis_hash_destroy(&rh); + if (bad) { + str_free_dup(&snap); + return -1; + } + + ml->checkpoint = g_new0(__typeof(*ml->checkpoint), 1); + ml->checkpoint->pending = pending && snap.len; + if (ml->checkpoint->pending) + ml->checkpoint->snapshot = snap; + else + str_free_dup(&snap); + } + return 0; +} + +struct redis_parsed_record { + JsonParser *json; + bencode_buffer_t benc; + bool benc_valid; + const ng_parser_t *parser; +}; + +static const char *redis_parse_record(const str *record, parser_arg *root, + struct redis_parsed_record *out) +{ + ZERO(*out); + if (!record->len) + return "empty record"; + + if (record->s[0] == '{') { + out->json = json_parser_new(); + if (!json_parser_load_from_data(out->json, record->s, record->len, NULL)) + return "could not parse JSON data"; + JsonNode *json_root = json_parser_get_root(out->json); + if (!json_root) + return "could not read JSON data"; + root->json = json_root; + redis_parser = out->parser = &ng_parser_json; + return NULL; + } + + if (record->s[0] == 'd') { + if (bencode_buffer_init(&out->benc)) + return "failed to initialise bencode buffer"; + out->benc_valid = true; + bencode_item_t *benc_root = bencode_decode_expect_str(&out->benc, record, + BENCODE_DICTIONARY); + if (!benc_root) + return "failed to decode bencode dictionary"; + root->benc = benc_root; + redis_parser = out->parser = &ng_parser_native; + return NULL; + } + + return "Unrecognised serial format"; +} + +static void redis_parsed_record_free(struct redis_parsed_record *p) { + if (p->json) + g_object_unref(p->json); + if (p->benc_valid) + bencode_buffer_free(&p->benc); +} + static void json_restore_call(struct redis *r, const str *callid, bool foreign) { redisReply* rr_jsonStr; struct redis_hash call; @@ -2084,10 +2240,7 @@ static void json_restore_call(struct redis *r, const str *callid, bool foreign) const char *err = 0; int i; atomic64 a64; - JsonNode *json_root = NULL; - JsonParser *parser = NULL; - bencode_item_t *benc_root = NULL; - bencode_buffer_t buf = {0}; + struct redis_parsed_record parsed = {0}; mutex_lock(&r->lock); rr_jsonStr = redis_get(r, REDIS_REPLY_STRING, "GET " PB, PBSTR(callid)); @@ -2102,35 +2255,9 @@ static void json_restore_call(struct redis *r, const str *callid, bool foreign) parser_arg root = {0}; - if (rr_jsonStr->str[0] == '{') { - parser = json_parser_new(); - err = "could not parse JSON data"; - if (!json_parser_load_from_data (parser, rr_jsonStr->str, -1, NULL)) - goto err1; - json_root = json_parser_get_root(parser); - err = "could not read JSON data"; - if (!json_root) - goto err1; - root.json = json_root; - redis_parser = &ng_parser_json; - } - else if (rr_jsonStr->str[0] == 'd') { - int ret = bencode_buffer_init(&buf); - err = "failed to initialise bencode buffer"; - if (ret) - goto err1; - err = "failed to decode bencode dictionary"; - benc_root = bencode_decode_expect_str(&buf, &STR_LEN(rr_jsonStr->str, rr_jsonStr->len), - BENCODE_DICTIONARY); - if (!benc_root) - goto err1; - redis_parser = &ng_parser_native; - root.benc = benc_root; - } - else { - err = "Unrecognised serial format"; + err = redis_parse_record(&STR_LEN(rr_jsonStr->str, rr_jsonStr->len), &root, &parsed); + if (err) goto err1; - } c = call_get_or_create(callid, false); err = "failed to create call struct"; @@ -2225,6 +2352,12 @@ static void json_restore_call(struct redis *r, const str *callid, bool foreign) err = "failed to link maps"; if (json_link_maps(c, &maps, &sfds, root)) goto err8; + if (redis_restore_checkpoints(c, root)) { + /* auxiliary state: an unreadable payload disables rollback rather than + * discarding an otherwise usable call */ + call_checkpoint_free_all(c); + ilog(LOG_WARNING, "Ignoring invalid checkpoint data while restoring call"); + } // presence of this key determines whether we were recording at all if (!redis_hash_get_str(&s, &call, "recording_meta_prefix")) { @@ -2263,15 +2396,13 @@ static void json_restore_call(struct redis *r, const str *callid, bool foreign) err4: json_destroy_list(&tags); err3: - json_destroy_hash(&call); + redis_hash_destroy(&call); err2: rwlock_unlock_w(&c->master_lock); err1: - if (parser) - g_object_unref (parser); + redis_parsed_record_free(&parsed); if (rr_jsonStr) - freeReplyObject(rr_jsonStr); - bencode_buffer_free(&buf); + freeReplyObject(rr_jsonStr); if (err) { mutex_lock(&r->lock); if (r->ctx && r->ctx->err) @@ -2458,6 +2589,23 @@ int redis_restore(struct redis *r, bool foreign, int db) { #define JSON_SET_SIMPLE_CSTR(a,d) parser->dict_add_str_dup(inner, a, STR_PTR(d)) #define JSON_SET_SIMPLE_STR(a,d) parser->dict_add_str_dup(inner, a, d) +void redis_encode_codec_store(const ng_parser_t *parser, parser_arg list, + const struct codec_store *store) +{ + char tmp[1024]; + for (__auto_type l = store->codec_prefs.head; l; l = l->next) { + rtp_payload_type *pt = l->data; + size_t len = rtpe_snprintf(tmp, sizeof(tmp), "%u/" STR_FORMAT "/%u/" STR_FORMAT + "/%i/%i/" STR_FORMAT "/" STR_FORMAT, + pt->payload_type, STR_FMT(&pt->encoding), pt->clock_rate, + STR_FMT(&pt->encoding_parameters), pt->bitrate, pt->ptime, + STR_FMT(&pt->format_parameters), STR_FMT(&pt->codec_opts)); + char encoded[len * 3 + 1]; + str value = parser->escape(encoded, tmp, len); + parser->list_add_str_dup(list, &value); + } +} + static void json_update_crypto_params(const ng_parser_t *parser, parser_arg inner, const char *key, struct crypto_params *p) { @@ -2476,9 +2624,8 @@ static void json_update_crypto_params(const ng_parser_t *parser, parser_arg inne JSON_SET_NSTRING_LEN("%s-mki", key, p->mki_len, (char *) p->mki); } -static int json_update_sdes_params(const ng_parser_t *parser, parser_arg inner, const char *pref, - unsigned int unique_id, - const char *k, sdes_q *q) +int redis_encode_sdes_params(const ng_parser_t *parser, parser_arg inner, const char *k, + const sdes_q *q) { unsigned int iter = 0; char keybuf[32]; @@ -2501,8 +2648,7 @@ static int json_update_sdes_params(const ng_parser_t *parser, parser_arg inner, return 0; } -static void json_update_dtls_fingerprint(const ng_parser_t *parser, parser_arg inner, const char *pref, - unsigned int unique_id, +void redis_encode_dtls_fingerprint(const ng_parser_t *parser, parser_arg inner, const struct dtls_fingerprint *f) { if (!f->hash_func) @@ -2512,11 +2658,30 @@ static void json_update_dtls_fingerprint(const ng_parser_t *parser, parser_arg i JSON_SET_SIMPLE_LEN("fingerprint", sizeof(f->digest), (char *) f->digest); } +static void json_update_detected_endpoints(const ng_parser_t *parser, parser_arg inner, + const struct packet_stream *ps) +{ + /* NSTRING: the key is built at runtime, so the parser must duplicate it */ + for (unsigned int i = 0; i < G_N_ELEMENTS(ps->detected_endpoints); i++) + JSON_SET_NSTRING_CSTR("detected_endpoint-%u", i, + ps->detected_endpoints[i].address.family + ? endpoint_print_buf(&ps->detected_endpoints[i]) : ""); +} + /** * encodes the few (k,v) pairs for one call under one json structure */ -static str redis_encode_json(ng_parser_ctx_t *ctx, call_t *c, void **to_free) { + +// scope = write only these monologues' state, plus the state only a rollback +// reads. NULL writes the whole call, which is what the Redis record wants. +static bool ml_in_scope(struct call_monologue * const *scope, const struct call_monologue *ml) { + return !scope || ml == scope[0] || ml == scope[1]; +} + +static str redis_encode_json(ng_parser_ctx_t *ctx, call_t *c, void **to_free, + struct call_monologue * const *scope) +{ char tmp[128]; const ng_parser_t *parser = ctx->parser; @@ -2524,9 +2689,10 @@ static str redis_encode_json(ng_parser_ctx_t *ctx, call_t *c, void **to_free) { parser_arg root = parser->dict(ctx); { - parser_arg inner = parser->dict_add_dict(root, "json"); + parser_arg inner = {0}; - { + if (!scope) { + inner = parser->dict_add_dict(root, "json"); JSON_SET_SIMPLE("created","%" PRId64, c->created); JSON_SET_SIMPLE("destroyed","%" PRId64, c->destroyed); JSON_SET_SIMPLE("last_signal","%" PRId64, c->last_signal_us); @@ -2557,7 +2723,25 @@ static str redis_encode_json(ng_parser_ctx_t *ctx, call_t *c, void **to_free) { JSON_SET_SIMPLE_STR("recording_random_tag", &c->recording_random_tag); } - for (__auto_type l = c->stream_fds.head; l; l = l->next) { + for (__auto_type l = scope ? NULL : c->monologues.head; l; l = l->next) { + const struct call_monologue *ml = l->data; + if (!ml->checkpoint) + continue; + snprintf(tmp, sizeof(tmp), "checkpoint-%u", ml->unique_id); + inner = parser->dict_add_dict_dup(root, tmp); + JSON_SET_SIMPLE("pending", "%i", ml->checkpoint->pending ? 1 : 0); + if (ml->checkpoint->snapshot.len) { + /* nested as a string; heap buffer rather than a VLA, as escape() can + * need up to 3x the input */ + char *enc = g_malloc_n(ml->checkpoint->snapshot.len + 1, 3); + str encs = parser->escape(enc, ml->checkpoint->snapshot.s, + ml->checkpoint->snapshot.len); + parser->dict_add_str_dup(inner, "snapshot", &encs); + g_free(enc); + } + } + + for (__auto_type l = scope ? NULL : c->stream_fds.head; l; l = l->next) { stream_fd *sfd = l->data; snprintf(tmp, sizeof(tmp), "sfd-%u", sfd->unique_id); @@ -2579,6 +2763,9 @@ static str redis_encode_json(ng_parser_ctx_t *ctx, call_t *c, void **to_free) { for (__auto_type l = c->streams.head; l; l = l->next) { struct packet_stream *ps = l->data; + if (!ps->media || !ml_in_scope(scope, ps->media->monologue)) + continue; + LOCK(&ps->lock); snprintf(tmp, sizeof(tmp), "stream-%u", ps->unique_id); @@ -2592,6 +2779,17 @@ static str redis_encode_json(ng_parser_ctx_t *ctx, call_t *c, void **to_free) { JSON_SET_SIMPLE("component","%u",ps->component); JSON_SET_SIMPLE_CSTR("endpoint",endpoint_print_buf(&ps->endpoint)); JSON_SET_SIMPLE_CSTR("advertised_endpoint",endpoint_print_buf(&ps->advertised_endpoint)); + if (scope) { + JSON_SET_SIMPLE_CSTR("learned_endpoint", + ps->learned_endpoint.address.family + ? endpoint_print_buf(&ps->learned_endpoint) : ""); + JSON_SET_SIMPLE_CSTR("last_local_endpoint", + ps->last_local_endpoint.address.family + ? endpoint_print_buf(&ps->last_local_endpoint) : ""); + JSON_SET_SIMPLE("ep_detect_signal", "%" PRId64, ps->ep_detect_signal); + JSON_SET_SIMPLE("el_flags", "%u", ps->el_flags); + json_update_detected_endpoints(parser, inner, ps); + } JSON_SET_SIMPLE("stats-packets","%" PRIu64, atomic64_get_na(&ps->stats_in->packets)); JSON_SET_SIMPLE("stats-bytes","%" PRIu64, atomic64_get_na(&ps->stats_in->bytes)); JSON_SET_SIMPLE("stats-errors","%" PRIu64, atomic64_get_na(&ps->stats_in->errors)); @@ -2606,26 +2804,33 @@ static str redis_encode_json(ng_parser_ctx_t *ctx, call_t *c, void **to_free) { JSON_ADD_LIST_STRING("%u", sfd->unique_id); } - snprintf(tmp, sizeof(tmp), "rtp_sinks-%u", ps->unique_id); - inner = parser->dict_add_list_dup(root, tmp); - for (__auto_type k = ps->rtp_sinks.head; k; k = k->next) { - struct sink_handler *sh = k->data; - struct packet_stream *sink = sh->sink; - JSON_ADD_LIST_STRING("%u", sink->unique_id); + if (!scope) { + snprintf(tmp, sizeof(tmp), "rtp_sinks-%u", ps->unique_id); + inner = parser->dict_add_list_dup(root, tmp); + for (__auto_type k = ps->rtp_sinks.head; k; k = k->next) { + struct sink_handler *sh = k->data; + struct packet_stream *sink = sh->sink; + JSON_ADD_LIST_STRING("%u", sink->unique_id); + } } - snprintf(tmp, sizeof(tmp), "rtcp_sinks-%u", ps->unique_id); - inner = parser->dict_add_list_dup(root, tmp); - for (__auto_type k = ps->rtcp_sinks.head; k; k = k->next) { - struct sink_handler *sh = k->data; - struct packet_stream *sink = sh->sink; - JSON_ADD_LIST_STRING("%u", sink->unique_id); + if (!scope) { + snprintf(tmp, sizeof(tmp), "rtcp_sinks-%u", ps->unique_id); + inner = parser->dict_add_list_dup(root, tmp); + for (__auto_type k = ps->rtcp_sinks.head; k; k = k->next) { + struct sink_handler *sh = k->data; + struct packet_stream *sink = sh->sink; + JSON_ADD_LIST_STRING("%u", sink->unique_id); + } } } // --- for streams.head for (__auto_type l = c->monologues.head; l; l = l->next) { struct call_monologue *ml = l->data; + if (!ml_in_scope(scope, ml)) + continue; + snprintf(tmp, sizeof(tmp), "tag-%u", ml->unique_id); inner = parser->dict_add_dict_dup(root, tmp); @@ -2680,22 +2885,31 @@ static str redis_encode_json(ng_parser_ctx_t *ctx, call_t *c, void **to_free) { JSON_SET_SIMPLE("sdp_session_rr", "%ld", ml->sdp_session_bandwidth.rr); if (ml->sdp_session_bandwidth.rs >= 0) JSON_SET_SIMPLE("sdp_session_rs", "%ld", ml->sdp_session_bandwidth.rs); + if (ml->sdp_session_bandwidth.tias >= 0) + JSON_SET_SIMPLE("sdp_session_tias", "%ld", ml->sdp_session_bandwidth.tias); + if (ml->last_out_sdp && ml->last_out_sdp->len) + JSON_SET_SIMPLE_LEN("last_out_sdp", ml->last_out_sdp->len, + ml->last_out_sdp->str); } GList *k = g_hash_table_get_values(ml->associated_tags); - snprintf(tmp, sizeof(tmp), "associated_tags-%u", ml->unique_id); - inner = parser->dict_add_list_dup(root, tmp); - for (GList *m = k; m; m = m->next) { - struct call_monologue *ml2 = m->data; - JSON_ADD_LIST_STRING("%u", ml2->unique_id); + if (!scope) { + snprintf(tmp, sizeof(tmp), "associated_tags-%u", ml->unique_id); + inner = parser->dict_add_list_dup(root, tmp); + for (GList *m = k; m; m = m->next) { + struct call_monologue *ml2 = m->data; + JSON_ADD_LIST_STRING("%u", ml2->unique_id); + } } g_list_free(k); - snprintf(tmp, sizeof(tmp), "tag_aliases-%u", ml->unique_id); - inner = parser->dict_add_list_dup(root, tmp); - for (__auto_type alias = ml->tag_aliases.head; alias; alias = alias->next) - JSON_ADD_LIST_STRING(STR_FORMAT, STR_FMT(alias->data)); + if (!scope) { + snprintf(tmp, sizeof(tmp), "tag_aliases-%u", ml->unique_id); + inner = parser->dict_add_list_dup(root, tmp); + for (__auto_type alias = ml->tag_aliases.head; alias; alias = alias->next) + JSON_ADD_LIST_STRING(STR_FORMAT, STR_FMT(alias->data)); + } snprintf(tmp, sizeof(tmp), "medias-%u", ml->unique_id); inner = parser->dict_add_list_dup(root, tmp); @@ -2708,20 +2922,22 @@ static str redis_encode_json(ng_parser_ctx_t *ctx, call_t *c, void **to_free) { for (__auto_type l = c->medias.head; l; l = l->next) { struct call_media *media = l->data; - if (!media) + if (!media || !ml_in_scope(scope, media->monologue)) continue; - /* store media subscriptions */ - snprintf(tmp, sizeof(tmp), "media-subscriptions-%u", media->unique_id); - inner = parser->dict_add_list_dup(root, tmp); - - IQUEUE_FOREACH(&media->media_subscriptions, ms) { - JSON_ADD_LIST_STRING("%u/%u/%u/%u/%u", - ms->media->unique_id, - ms->attrs.offer_answer, - ms->attrs.rtcp_only, - ms->attrs.egress, - ms->attrs.inject); + if (!scope) { + /* store media subscriptions */ + snprintf(tmp, sizeof(tmp), "media-subscriptions-%u", media->unique_id); + inner = parser->dict_add_list_dup(root, tmp); + + IQUEUE_FOREACH(&media->media_subscriptions, ms) { + JSON_ADD_LIST_STRING("%u/%u/%u/%u/%u", + ms->media->unique_id, + ms->attrs.offer_answer, + ms->attrs.rtcp_only, + ms->attrs.egress, + ms->attrs.inject); + } } snprintf(tmp, sizeof(tmp), "media-%u", media->unique_id); @@ -2749,38 +2965,87 @@ static str redis_encode_json(ng_parser_ctx_t *ctx, call_t *c, void **to_free) { JSON_SET_SIMPLE("bandwidth_rr","%ld", media->sdp_media_bandwidth.rr); if (media->sdp_media_bandwidth.rs >= 0) JSON_SET_SIMPLE("bandwidth_rs","%ld", media->sdp_media_bandwidth.rs); + if (media->sdp_media_bandwidth.ct >= 0) + JSON_SET_SIMPLE("bandwidth_ct","%ld", media->sdp_media_bandwidth.ct); + if (media->sdp_media_bandwidth.tias >= 0) + JSON_SET_SIMPLE("bandwidth_tias","%ld", media->sdp_media_bandwidth.tias); + + if (scope) { + if (media->tls_id.s) + JSON_SET_SIMPLE_STR("tls_id", &media->tls_id); + if (media->fp_hash_func) + JSON_SET_SIMPLE_CSTR("preferred_hash_func", + media->fp_hash_func->name); + if (media->endpoint_map) + JSON_SET_SIMPLE("endpoint_map", "%u", + media->endpoint_map->unique_id); + + unsigned int num_cands = 0; + for (__auto_type m = media->ice_candidates.head; m; m = m->next) + num_cands++; + JSON_SET_SIMPLE("num_ice_candidates", "%u", num_cands); + JSON_SET_SIMPLE("had_ice", "%i", media->ice_agent ? 1 : 0); + if (media->ice_agent) { + JSON_SET_SIMPLE_STR("ice_ufrag_local", &media->ice_agent->ufrag[0]); + JSON_SET_SIMPLE_STR("ice_ufrag_remote", &media->ice_agent->ufrag[1]); + JSON_SET_SIMPLE_STR("ice_pwd_local", &media->ice_agent->pwd[0]); + JSON_SET_SIMPLE_STR("ice_pwd_remote", &media->ice_agent->pwd[1]); + } + } - json_update_sdes_params(parser, inner, "media", media->unique_id, "sdes_in", - &media->sdes_in); - json_update_sdes_params(parser, inner, "media", media->unique_id, "sdes_out", - &media->sdes_out); - json_update_dtls_fingerprint(parser, inner, "media", media->unique_id, &media->fingerprint); + redis_encode_sdes_params(parser, inner, "sdes_in", &media->sdes_in); + redis_encode_sdes_params(parser, inner, "sdes_out", &media->sdes_out); + redis_encode_dtls_fingerprint(parser, inner, &media->fingerprint); } - snprintf(tmp, sizeof(tmp), "streams-%u", media->unique_id); - inner = parser->dict_add_list_dup(root, tmp); - for (__auto_type m = media->streams.head; m; m = m->next) { - struct packet_stream *ps = m->data; - JSON_ADD_LIST_STRING("%u", ps->unique_id); + if (scope) { + unsigned int ci = 0; + for (__auto_type m = media->ice_candidates.head; m; m = m->next, ci++) { + const struct ice_candidate *cand = m->data; + snprintf(tmp, sizeof(tmp), "ice_candidate-%u-%u", media->unique_id, ci); + inner = parser->dict_add_dict_dup(root, tmp); + JSON_SET_SIMPLE_STR("foundation", &cand->foundation); + JSON_SET_SIMPLE("component", "%lu", (unsigned long) cand->component_id); + JSON_SET_SIMPLE_CSTR("transport", + cand->transport ? cand->transport->name : ""); + JSON_SET_SIMPLE("priority", "%lu", (unsigned long) cand->priority); + JSON_SET_SIMPLE("type", "%u", cand->type); + JSON_SET_SIMPLE_STR("ufrag", &cand->ufrag); + JSON_SET_SIMPLE_CSTR("endpoint", + cand->endpoint.address.family + ? endpoint_print_buf(&cand->endpoint) : ""); + JSON_SET_SIMPLE_CSTR("related", + cand->related.address.family + ? endpoint_print_buf(&cand->related) : ""); + } } - snprintf(tmp, sizeof(tmp), "maps-%u", media->unique_id); - inner = parser->dict_add_list_dup(root, tmp); - for (__auto_type m = media->endpoint_maps.head; m; m = m->next) { - struct endpoint_map *ep = m->data; - JSON_ADD_LIST_STRING("%u", ep->unique_id); + if (!scope) { + snprintf(tmp, sizeof(tmp), "streams-%u", media->unique_id); + inner = parser->dict_add_list_dup(root, tmp); + for (__auto_type m = media->streams.head; m; m = m->next) { + struct packet_stream *ps = m->data; + JSON_ADD_LIST_STRING("%u", ps->unique_id); + } + } + + if (!scope) { + snprintf(tmp, sizeof(tmp), "maps-%u", media->unique_id); + inner = parser->dict_add_list_dup(root, tmp); + for (__auto_type m = media->endpoint_maps.head; m; m = m->next) { + struct endpoint_map *ep = m->data; + JSON_ADD_LIST_STRING("%u", ep->unique_id); + } } snprintf(tmp, sizeof(tmp), "payload_types-%u", media->unique_id); inner = parser->dict_add_list_dup(root, tmp); - for (__auto_type m = media->codecs.codec_prefs.head; m; m = m->next) { - rtp_payload_type *pt = m->data; - JSON_ADD_LIST_STRING("%u/" STR_FORMAT "/%u/" STR_FORMAT "/%i/%i/" - STR_FORMAT "/" STR_FORMAT, - pt->payload_type, STR_FMT(&pt->encoding), - pt->clock_rate, STR_FMT(&pt->encoding_parameters), - pt->bitrate, pt->ptime, STR_FMT(&pt->format_parameters), - STR_FMT(&pt->codec_opts)); + redis_encode_codec_store(parser, inner, &media->codecs); + + if (scope) { + snprintf(tmp, sizeof(tmp), "offered_payload_types-%u", media->unique_id); + inner = parser->dict_add_list_dup(root, tmp); + redis_encode_codec_store(parser, inner, &media->offered_codecs); } // SSRC table dump @@ -2804,7 +3069,7 @@ static str redis_encode_json(ng_parser_ctx_t *ctx, call_t *c, void **to_free) { } } // --- for medias.head - for (__auto_type l = c->endpoint_maps.head; l; l = l->next) { + for (__auto_type l = scope ? NULL : c->endpoint_maps.head; l; l = l->next) { struct endpoint_map *ep = l->data; snprintf(tmp, sizeof(tmp), "map-%u", ep->unique_id); @@ -2837,6 +3102,415 @@ static str redis_encode_json(ng_parser_ctx_t *ctx, call_t *c, void **to_free) { } +str redis_snapshot_encode(call_t *c, struct call_monologue *ml) { + struct call_monologue *scope[2] = { ml, ml }; + ng_parser_ctx_t ctx; + bencode_buffer_t bbuf; + // never leaves the daemon, so the format is ours to pick + ng_parser_native.init(&ctx, &bbuf); + + void *to_free = NULL; + str encoded = redis_encode_json(&ctx, c, &to_free, scope); + str out = STR_NULL; + if (encoded.len) + out = str_dup_str(&encoded); + + g_free(to_free); + bencode_buffer_free(ctx.buffer); + return out; +} + +void redis_snapshot_free(str *snap) { + str_free_dup(snap); +} + +static stream_fd *snapshot_find_sfd(call_t *c, unsigned int id) { + for (__auto_type l = c->stream_fds.head; l; l = l->next) { + stream_fd *sfd = l->data; + if (sfd->unique_id == id) + return sfd; + } + return NULL; +} + +static struct endpoint_map *snapshot_find_map(call_t *c, unsigned int id) { + for (__auto_type l = c->endpoint_maps.head; l; l = l->next) { + struct endpoint_map *map = l->data; + if (map->unique_id == id) + return map; + } + return NULL; +} + +struct snapshot_sfd_iter { + call_t *call; + stream_fd_q *out; +}; + +static const char *snapshot_sfd_iter(str *val, unsigned int idx, helper_arg arg) { + struct snapshot_sfd_iter *args = arg.generic; + str *sid = redis_parser->unescape(val->s, val->len); + int id = str_to_i(sid, -1); + g_free(sid); + if (id < 0) + return NULL; + stream_fd *sfd = snapshot_find_sfd(args->call, (unsigned int) id); + if (!sfd) + return NULL; + stream_fd_inc(sfd); + t_queue_push_tail(args->out, sfd); + return NULL; +} + +static void snapshot_apply_stream(call_t *c, struct packet_stream *ps, + const struct redis_hash *rh, parser_arg root) +{ + int64_t iv; + str s; + + dtls_shutdown(ps); + + int64_t sfd_id = -1; + redis_hash_get_int64_t(&sfd_id, rh, "sfd"); + /* a socket that no longer exists, or was never bound here after a takeover: + * keep the live binding, as restoring an unbound one would silence the call */ + stream_fd *want_sfd = sfd_id >= 0 ? snapshot_find_sfd(c, (unsigned int) sfd_id) : NULL; + bool live_is_usable = ps->selected_sfd && ps->selected_sfd->socket.local.port; + bool want_is_usable = want_sfd && want_sfd->socket.local.port; + + if (want_is_usable && want_sfd != ps->selected_sfd) { + char lkey[64]; + snprintf(lkey, sizeof(lkey), "stream_sfds-%u", ps->unique_id); + parser_arg list = redis_parser->dict_get_expect(root, lkey, BENCODE_LIST); + stream_fd_q restored = TYPED_GQUEUE_INIT; + if (list.gen) { + struct snapshot_sfd_iter args = { .call = c, .out = &restored }; + redis_parser->list_iter(redis_parser, list, snapshot_sfd_iter, NULL, &args); + } + if (!restored.length) { + stream_fd_inc(want_sfd); + t_queue_push_tail(&restored, want_sfd); + } + t_queue_clear_full(&ps->sfds, stream_fd_dec); + ps->sfds = restored; + ps->selected_sfd = want_sfd; + } + else if (!live_is_usable && want_sfd) { + ps->selected_sfd = want_sfd; + } + + // propagated rather than ignored, so a malformed snapshot isn't applied piecemeal + if (redis_decode_stream_fields(ps, rh)) + return; + if (!redis_hash_get_str(&s, rh, "learned_endpoint") && s.len) + redis_hash_get_endpoint(&ps->learned_endpoint, rh, "learned_endpoint"); + if (!redis_hash_get_str(&s, rh, "last_local_endpoint") && s.len) + redis_hash_get_endpoint(&ps->last_local_endpoint, rh, "last_local_endpoint"); + + for (unsigned int i = 0; i < G_N_ELEMENTS(ps->detected_endpoints); i++) { + /* an absent endpoint is an empty string, which would parse as 0.0.0.0:0 */ + if (!redis_hash_get_str_f(&s, rh, "detected_endpoint-%u", i) && s.len) + redis_hash_get_endpoint_f(&ps->detected_endpoints[i], rh, + "detected_endpoint-%u", i); + else + ZERO(ps->detected_endpoints[i]); + } + + if (!redis_hash_get_int64_t(&iv, rh, "ep_detect_signal")) + ps->ep_detect_signal = iv; + if (!redis_hash_get_int64_t(&iv, rh, "el_flags")) + ps->el_flags = iv; + + /* only recorded when a suite is in use, so an absent one means the rejected + * offer's context has to go, or a plain-RTP media keeps SRTP configured */ + if (redis_hash_get_str(&s, rh, "-crypto_suite")) + crypto_reset(&ps->crypto); +} + +static void snapshot_apply_media_crypto(struct call_media *m, const struct redis_hash *rh) { + str s; + + /* cleared first: the fingerprint decoder reports absence as success, and SDES appends */ + crypto_params_sdes_queue_clear(&m->sdes_in); + crypto_params_sdes_queue_clear(&m->sdes_out); + redis_decode_sdes_params(&m->sdes_in, rh, "sdes_in"); + redis_decode_sdes_params(&m->sdes_out, rh, "sdes_out"); + + if (!redis_hash_get_str(&s, rh, "hash_func")) + redis_decode_dtls_fingerprint(&m->fingerprint, rh); + else + ZERO(m->fingerprint); +} + +static void snapshot_apply_media_codecs(struct call_media *m, parser_arg root) { + static const struct { + const char *key; + size_t offset; + } stores[] = { + { "payload_types-%u", G_STRUCT_OFFSET(struct call_media, codecs) }, + { "offered_payload_types-%u", G_STRUCT_OFFSET(struct call_media, offered_codecs) }, + }; + + for (unsigned int i = 0; i < G_N_ELEMENTS(stores); i++) { + char key[64]; + snprintf(key, sizeof(key), stores[i].key, m->unique_id); + parser_arg list = redis_parser->dict_get_expect(root, key, BENCODE_LIST); + if (!list.gen) + continue; + struct codec_store *cs = &G_STRUCT_MEMBER(struct codec_store, m, stores[i].offset); + codec_store_cleanup(cs); + codec_store_init(cs, m); + redis_decode_codec_store(redis_parser, list, cs); + } + + codec_handlers_free(m); +} + +static void snapshot_apply_media_ice(struct call_media *m, const struct redis_hash *rh, + parser_arg root) +{ + str s; + int64_t iv; + + /* not rewound: applying the offer already reset the agent, so restoring the + * accepted credentials lets connectivity checks rebuild the state */ + int64_t had_ice = 0; + redis_hash_get_int64_t(&had_ice, rh, "had_ice"); + if (!had_ice) { + ice_candidates_free(&m->ice_candidates); + ice_shutdown(&m->ice_agent); + return; + } + + /* The agent keeps these, so they must outlive the hash they are read from. */ + str ufrag[2] = {STR_NULL, STR_NULL}, pwd[2] = {STR_NULL, STR_NULL}; + static const char *const ice_keys[4] = { + "ice_ufrag_local", "ice_ufrag_remote", "ice_pwd_local", "ice_pwd_remote", + }; + str *ice_vals[4] = { &ufrag[0], &ufrag[1], &pwd[0], &pwd[1] }; + for (unsigned int i = 0; i < G_N_ELEMENTS(ice_keys); i++) { + str raw; + if (!redis_hash_get_str(&raw, rh, ice_keys[i])) + *ice_vals[i] = call_str_cpy(&raw); + } + + candidate_q cands = TYPED_GQUEUE_INIT; + int64_t num_cands = 0; + redis_hash_get_int64_t(&num_cands, rh, "num_ice_candidates"); + for (int64_t i = 0; i < num_cands; i++) { + char ckey[64]; + snprintf(ckey, sizeof(ckey), "ice_candidate-%u-%lld", m->unique_id, (long long) i); + struct redis_hash ch; + if (json_get_hash(&ch, ckey, -1, root)) + continue; + struct ice_candidate *cand = g_new0(__typeof(*cand), 1); + if (!redis_hash_get_str(&s, &ch, "foundation")) + cand->foundation = call_str_cpy(&s); + if (!redis_hash_get_str(&s, &ch, "ufrag")) + cand->ufrag = call_str_cpy(&s); + if (!redis_hash_get_str(&s, &ch, "transport")) + cand->transport = get_socket_type(&s); + if (!redis_hash_get_int64_t(&iv, &ch, "component")) + cand->component_id = iv; + if (!redis_hash_get_int64_t(&iv, &ch, "priority")) + cand->priority = iv; + if (!redis_hash_get_int64_t(&iv, &ch, "type")) + cand->type = iv; + /* an absent endpoint is an empty string; parsing it would yield 0.0.0.0:0 */ + if (!redis_hash_get_str(&s, &ch, "endpoint") && s.len) + redis_hash_get_endpoint(&cand->endpoint, &ch, "endpoint"); + if (!redis_hash_get_str(&s, &ch, "related") && s.len) + redis_hash_get_endpoint(&cand->related, &ch, "related"); + t_queue_push_tail(&cands, cand); + redis_hash_destroy(&ch); + } + + /* the media keeps its own list too: it's what the record and any regenerated + * SDP are built from, so restoring only the agent loses them */ + ice_candidates_free(&m->ice_candidates); + for (__auto_type l = cands.head; l; l = l->next) { + struct ice_candidate *copy = g_new0(__typeof(*copy), 1); + *copy = *(struct ice_candidate *) l->data; + t_queue_push_tail(&m->ice_candidates, copy); + } + ice_agent_init(&m->ice_agent, m); + ice_rollback(m->ice_agent, ufrag, pwd, &cands); + ice_candidates_free(&cands); +} + +static void snapshot_apply_media(call_t *c, struct call_media *m, + const struct redis_hash *rh, parser_arg root) +{ + int64_t iv; + str s; + + if (redis_decode_media_fields(m, rh)) + return; + + // assigned unconditionally: these are written only when set, so an absent key + // means the rejected offer put it there and it has to go + m->protocol_str = !redis_hash_get_str(&s, rh, "protocol") ? call_str_cpy(&s) : STR_NULL; + m->tls_id = !redis_hash_get_str(&s, rh, "tls_id") ? call_str_cpy(&s) : STR_NULL; + m->fp_hash_func = !redis_hash_get_str(&s, rh, "preferred_hash_func") + ? dtls_find_hash_func(&s) : NULL; + m->endpoint_map = !redis_hash_get_int64_t(&iv, rh, "endpoint_map") + ? snapshot_find_map(c, (unsigned int) iv) : NULL; + m->sdp_media_bandwidth.ct = !redis_hash_get_int64_t(&iv, rh, "bandwidth_ct") ? iv : -1; + m->sdp_media_bandwidth.tias = !redis_hash_get_int64_t(&iv, rh, "bandwidth_tias") ? iv : -1; + + // a media always has one, so fall back to the default rather than to nothing + if (redis_hash_get_str(&s, rh, "logical_intf") + || !(m->logical_intf = get_logical_interface(&s, m->desired_family, 0))) + m->logical_intf = get_logical_interface(NULL, m->desired_family, 0); + + if (!redis_hash_get_int64_t(&iv, rh, "media_flags")) + atomic64_set_na(&m->media_flags, (uint64_t) iv); + + snapshot_apply_media_crypto(m, rh); + + snapshot_apply_media_codecs(m, root); + + snapshot_apply_media_ice(m, rh, root); +} + +static void snapshot_apply_monologue(struct call_monologue *ml, const struct redis_hash *rh) { + int64_t iv; + str s; + + redis_decode_monologue_sdp(ml, rh); + + if (!redis_hash_get_str(&s, rh, "desired_family")) + ml->desired_family = get_socket_family_rfc(&s); + if (!redis_hash_get_str(&s, rh, "logical_intf") + && !(ml->logical_intf = get_logical_interface(&s, ml->desired_family, 0))) + ml->logical_intf = get_logical_interface(NULL, ml->desired_family, 0); + if (!redis_hash_get_int64_t(&iv, rh, "ml_flags")) + atomic64_set_na(&ml->ml_flags, (uint64_t) iv); + if (ml->last_out_sdp) + g_string_free(ml->last_out_sdp, TRUE); + ml->last_out_sdp = !redis_hash_get_str(&s, rh, "last_out_sdp") + ? g_string_new_len(s.s, s.len) : NULL; +} + +static const char *snapshot_count_iter(str *val, unsigned int idx, helper_arg arg) { + unsigned int *n = arg.generic; + (*n)++; + return NULL; +} + +static unsigned int snapshot_medias_len(struct call_monologue *ml, parser_arg root) { + char key[64]; + snprintf(key, sizeof(key), "medias-%u", ml->unique_id); + parser_arg list = redis_parser->dict_get_expect(root, key, BENCODE_LIST); + if (!list.gen) + return ml->medias->len; + unsigned int n = 0; + redis_parser->list_iter(redis_parser, list, snapshot_count_iter, NULL, &n); + return n; +} + +static void snapshot_apply_medias(call_t *c, struct call_monologue *ml, parser_arg root) { + for (unsigned int j = 0; j < ml->medias->len; j++) { + struct call_media *m = ml->medias->pdata[j]; + struct redis_hash rh; + if (!m || json_get_hash(&rh, "media", m->unique_id, root)) + continue; + snapshot_apply_media(c, m, &rh, root); + redis_hash_destroy(&rh); + } +} + +static void snapshot_apply_monologues(struct call_monologue *ml, parser_arg root) { + struct redis_hash rh; + if (!json_get_hash(&rh, "tag", ml->unique_id, root)) { + snapshot_apply_monologue(ml, &rh); + redis_hash_destroy(&rh); + } + unsigned int keep = snapshot_medias_len(ml, root); + for (unsigned int j = keep; j < ml->medias->len; j++) + call_media_stop(ml->medias->pdata[j]); + if (keep < ml->medias->len) + t_ptr_array_set_size(ml->medias, keep); +} + +static void snapshot_apply_streams(call_t *c, struct call_monologue *ml, parser_arg root) { + for (unsigned int j = 0; j < ml->medias->len; j++) { + struct call_media *m = ml->medias->pdata[j]; + if (!m) + continue; + for (__auto_type l = m->streams.head; l; l = l->next) { + struct packet_stream *ps = l->data; + struct redis_hash rh; + if (json_get_hash(&rh, "stream", ps->unique_id, root)) + continue; + snapshot_apply_stream(c, ps, &rh, root); + redis_hash_destroy(&rh); + __init_stream(ps); + } + } +} + +bool redis_snapshot_apply(call_t *c, struct call_monologue *a, struct call_monologue *b) { + struct call_monologue *mls[2] = { a, b }; + struct redis_parsed_record parsed[2] = {0}; + parser_arg root[2] = {0}; + bool live[2] = { false, false }; + bool ok = false; + + for (unsigned int i = 0; i < G_N_ELEMENTS(mls); i++) { + struct call_monologue *ml = mls[i]; + if (!ml || !ml->checkpoint || !ml->checkpoint->pending) + continue; + if (!ml->checkpoint->snapshot.len) + continue; + if (redis_parse_record(&ml->checkpoint->snapshot, &root[i], &parsed[i])) + goto out; + live[i] = true; + } + + if (!live[0] && !live[1]) + goto out; + + // order matters: monologues need the medias, subscriptions need the + // monologues, and initialising the streams needs both + for (unsigned int i = 0; i < G_N_ELEMENTS(mls); i++) { + if (!live[i]) + continue; + redis_parser = parsed[i].parser; + snapshot_apply_medias(c, mls[i], root[i]); + } + for (unsigned int i = 0; i < G_N_ELEMENTS(mls); i++) { + if (!live[i]) + continue; + redis_parser = parsed[i].parser; + snapshot_apply_monologues(mls[i], root[i]); + } + + update_init_monologue_subscribers(a, OP_OFFER); + update_init_monologue_subscribers(b, OP_ANSWER); + + for (unsigned int i = 0; i < G_N_ELEMENTS(mls); i++) { + if (!live[i]) + continue; + redis_parser = parsed[i].parser; + snapshot_apply_streams(c, mls[i], root[i]); + } + + for (unsigned int i = 0; i < G_N_ELEMENTS(mls); i++) { + if (!live[i]) + continue; + redis_snapshot_free(&mls[i]->checkpoint->snapshot); + mls[i]->checkpoint->pending = false; + } + ok = true; + +out: + for (unsigned int i = 0; i < G_N_ELEMENTS(mls); i++) + redis_parsed_record_free(&parsed[i]); + return ok; +} + + void redis_update_onekey(call_t *c, struct redis *r) { unsigned int redis_expires_s; @@ -2867,7 +3541,7 @@ void redis_update_onekey(call_t *c, struct redis *r) { redis_format_parsers[rtpe_config.redis_format]->init(&ctx, &bbuf); void *to_free = NULL; - str result = redis_encode_json(&ctx, c, &to_free); + str result = redis_encode_json(&ctx, c, &to_free, NULL); if (!result.len) goto err; diff --git a/docs/ng_control_protocol.md b/docs/ng_control_protocol.md index 4eb291b1d..6ace64d6d 100644 --- a/docs/ng_control_protocol.md +++ b/docs/ng_control_protocol.md @@ -845,6 +845,12 @@ Optionally included keys are: dictionary. The response dictionary may also contain the optional key `message` with an explanatory string. No other key is required in the response dictionary. + * `rollback` + + Indicates that the controlling SIP proxy understands the `rollback` + message. If `rollback` is listed, *rtpengine* includes it in a + `supported` list in the response. + * `to-interface` Contains a string identifying the network interface pertaining to the @@ -1390,6 +1396,19 @@ Spaces in each string may be replaced by hyphens. address that has been learned before. If there's a mismatch, the packet will be dropped and not forwarded. +* `track state` + + Enables rollback checkpoints for the selected dialogue. Before applying an + `offer`, *rtpengine* records the media state from the last completed + offer/answer exchange. A successful `answer` commits the exchange and + discards the pending checkpoint. Once enabled, subsequent offers for the + dialogue are checkpointed without repeating the flag. The spelling + `track-state` is equivalent. + + Checkpointing is opt-in because a pending checkpoint retains media and + cryptographic configuration. Calls that do not use this flag do not retain + that state. + * `trickle ICE` Useful for `offer` messages when ICE is advertised to also advertise @@ -1806,8 +1825,10 @@ An example of a complete `offer` request dictionary could be (SDP body abbreviat "ICE": "force", "transport protocol": "RTP/SAVPF", "media address": "2001:d8::6f24:65b", "DTLS": "passive" } -A response message contains only the key `sdp` in addition to `result`, which contains the re-written -SDP body that the SIP proxy should insert into the SIP message. +A response message contains the key `sdp` in addition to `result`, which contains the re-written +SDP body that the SIP proxy should insert into the SIP message. If `supports` +requested a supported extension, the response can also contain a `supported` +list. Example response: @@ -1850,6 +1871,70 @@ the `direction` key in the `answer` message. The reply message is identical as in the `offer` reply. +## `rollback` Message + +The `rollback` message restores a dialogue to the media state from its last +completed offer/answer exchange without deleting the call. It is intended for +use when an SDP offer has already been applied by *rtpengine* but the remote +endpoint subsequently rejects the signalling transaction. The signalling +element must issue the message explicitly; *rtpengine* does not observe SIP +transaction outcomes. + +The request must contain `call-id`, `from-tag`, and `to-tag`. It may also +contain: + +* `via-branch` + + Selects a particular fork using the same dialogue matching rules as other + NG messages. + +The successful response contains `rolled-back`, set to `1` if a pending +checkpoint was restored or `0` if there was none outstanding. Repeating a +successful rollback is therefore safe and returns `rolled-back: 0`. + +Each side of a dialogue holds at most one outstanding checkpoint. Offers that +arrive before an exchange completes belong to the same uncommitted exchange and +keep the existing snapshot, so a rollback returns to the last completed +offer/answer rather than to an intermediate one. A signalling element should +not therefore issue a new offer for a dialogue while a rollback for it is still +in flight: the rollback restores the last completed state and the newer offer is +undone with it. + +Rollback restores addresses and ports, codecs and payload mappings, transport +profile, media direction, and SDES configuration including keys. ICE +credentials are restored and connectivity checks reconstruct candidate-pair +and nomination state. DTLS fingerprint, TLS ID, and setup/role configuration +are restored, but the live OpenSSL association is not serializable and must +perform a new handshake. + +State the rejected offer introduced is removed as well as overwritten. An offer +that upgraded a media to DTLS-SRTP, for example, leaves behind no TLS ID, +fingerprint or SRTP context once it has been rolled back. + +Where a call has been forked, the offering side is shared between the branches. +Its checkpoint is taken once, before the first uncommitted offer, so rolling +back one branch does not disturb what rolling back another has already +restored. + +Merging calls, as `connect` and `mesh` do, discards any outstanding checkpoint. +The merged call renumbers the state a snapshot refers to, so a rollback after a +merge reports none outstanding. + +Sockets and endpoint maps allocated for a rejected offer are not released by a +rollback. The media is returned to the sockets it was using, and the surplus is +reclaimed with the call. + +When calls are restored from Redis, checkpoint data is auxiliary: a checkpoint +that cannot be read is discarded in full while the call itself is restored +without rollback capability. + +Example request and response: + + { "command": "rollback", "call-id": "cfBXzDSZqhYNcXM", + "from-tag": "mS9rSAn0Cr", "to-tag": "yB3KjLa9" } + + { "result": "ok", "rolled-back": 1 } + ## `delete` Message The `delete` message must contain at least the keys `call-id` and `from-tag` and may optionally include diff --git a/include/call.h b/include/call.h index 470ff72db..067437b57 100644 --- a/include/call.h +++ b/include/call.h @@ -66,7 +66,7 @@ enum message_type { || (opmode == OP_UNSUBSCRIBE || opmode == OP_START_RECORDING) \ || (opmode == OP_STOP_RECORDING || opmode == OP_PAUSE_RECORDING) \ || (opmode == OP_INJECT_START || opmode == OP_INJECT_STOP) \ - || (opmode == OP_OTHER)) + || (opmode == OP_ROLLBACK || opmode == OP_OTHER)) #define IS_OP_DIRECTIONAL(opmode) \ ((opmode == OP_BLOCK_DTMF || opmode == OP_BLOCK_MEDIA) \ @@ -669,6 +669,8 @@ struct call_monologue { str moh_file; atomic64 ml_flags; + + struct call_checkpoint *checkpoint; }; TYPED_GHASHTABLE(str_ml_ht, str, struct call_monologue, str_hash, str_equal, NULL, NULL) @@ -961,6 +963,17 @@ void call_media_unkernelize(struct call_media *media, const char *reason); void __monologue_unconfirm(struct call_monologue *monologue, const char *); void __media_unconfirm(struct call_media *media, const char *); __attribute__((nonnull(1))) +/* one monologue's state from before an offer, held as a call record snapshot */ +struct call_checkpoint { + bool pending; + str snapshot; +}; + +void call_checkpoint_offer(call_t *, struct call_monologue *, struct call_monologue *, bool); +void call_checkpoint_answer(call_t *, struct call_monologue *, struct call_monologue *); +int call_checkpoint_rollback(call_t *, struct call_monologue *, struct call_monologue *); +void call_checkpoint_free_all(call_t *); + void update_init_monologue_subscribers(struct call_monologue *ml, enum ng_opmode opmode); int call_stream_address(GString *, struct packet_stream *ps, enum stream_address_format format, @@ -971,6 +984,7 @@ enum thread_looper_action call_timer(void); void __rtp_stats_update(rtp_stats_ht dst, struct codec_store *); bool __init_stream(struct packet_stream *ps); +void call_media_stop(struct call_media *); const rtp_payload_type *__rtp_stats_codec(struct call_media *m); diff --git a/include/call_flags.h b/include/call_flags.h index bbf5128a6..6129df446 100644 --- a/include/call_flags.h +++ b/include/call_flags.h @@ -288,6 +288,8 @@ RTPE_NG_FLAGS_STR_CASE_HT_PARAMS t38_no_iaf:1, t38_fec:1, supports_load_limit:1, + supports_rollback:1, + track_state:1, dtls_off:1, sdes_off:1, sdes_unencrypted_srtp:1, diff --git a/include/call_interfaces.h b/include/call_interfaces.h index 732b806c9..be15e484e 100644 --- a/include/call_interfaces.h +++ b/include/call_interfaces.h @@ -31,6 +31,7 @@ str call_query_udp(char **); const char *call_ping_ng(ng_command_ctx_t *ctx); const char *call_offer_ng(ng_command_ctx_t *, const char *); const char *call_answer_ng(ng_command_ctx_t *); +const char *call_rollback_ng(ng_command_ctx_t *); const char *call_delete_ng(ng_command_ctx_t *); const char *call_query_ng(ng_command_ctx_t *); const char *call_list_ng(ng_command_ctx_t *); diff --git a/include/control_ng.h b/include/control_ng.h index 54534be7b..7eaf91a83 100644 --- a/include/control_ng.h +++ b/include/control_ng.h @@ -5,6 +5,7 @@ X(OP_PING, "ping", "ping", "Ping", call_ping_ng) \ XA(OP_OFFER, "offer", "offer", "Offer", call_offer_ng) \ X(OP_ANSWER, "answer", "answer", "Answer", call_answer_ng) \ + X(OP_ROLLBACK, "rollback", "rollback", "Rollback", call_rollback_ng) \ X(OP_DELETE, "delete", "delete", "Delete", call_delete_ng) \ X(OP_QUERY, "query", "query", "Query", call_query_ng) \ X(OP_LIST, "list", "list", "List", call_list_ng) \ diff --git a/include/ice.h b/include/ice.h index 84708826e..3f7d4c080 100644 --- a/include/ice.h +++ b/include/ice.h @@ -159,6 +159,7 @@ void ice_update(struct ice_agent *, struct stream_params *, bool allow_restart); void ice_start(struct ice_agent *); void ice_shutdown(struct ice_agent **); void ice_restart(struct ice_agent *); +void ice_rollback(struct ice_agent *, const str [2], const str [2], const candidate_q *); void ice_candidates_free(candidate_q *); void ice_remote_candidates(candidate_q *, struct ice_agent *); diff --git a/include/redis.h b/include/redis.h index 550008ed1..e4039e096 100644 --- a/include/redis.h +++ b/include/redis.h @@ -11,6 +11,9 @@ #include "helpers.h" #include "call.h" #include "str.h" +#include "control_ng.h" +#include "crypto.h" +#include "dtls.h" #define REDIS_RESTORE_NUM_THREADS 4 @@ -77,6 +80,20 @@ struct redis_list { void **ptrs; }; +str redis_snapshot_encode(call_t *, struct call_monologue *); +void redis_snapshot_free(str *); +bool redis_snapshot_apply(call_t *, struct call_monologue *, struct call_monologue *); + +int redis_encode_sdes_params(const ng_parser_t *, parser_arg, const char *, const sdes_q *); +void redis_encode_dtls_fingerprint(const ng_parser_t *, parser_arg, + const struct dtls_fingerprint *); +int redis_decode_sdes_params(sdes_q *, const struct redis_hash *, const char *); +int redis_decode_dtls_fingerprint(struct dtls_fingerprint *, const struct redis_hash *); +int redis_hash_from_parser(struct redis_hash *, const ng_parser_t *, parser_arg); +void redis_hash_destroy(struct redis_hash *); +void redis_encode_codec_store(const ng_parser_t *, parser_arg, const struct codec_store *); +int redis_decode_codec_store(const ng_parser_t *, parser_arg, struct codec_store *); + extern struct redis *rtpe_redis; extern struct redis *rtpe_redis_write; diff --git a/t/Makefile b/t/Makefile index acf9644b4..c444c1e71 100644 --- a/t/Makefile +++ b/t/Makefile @@ -8,7 +8,11 @@ with_transcoding ?= yes include ../lib/flags.Makefile -PRELOAD_CFLAGS := $(CFLAGS) +# The preload shims are loaded by the test harness into the test scripts' own +# interpreter, so they must not carry the sanitizer runtime: in a sanitizer +# build that crashes the daemon tests on aarch64 during library init. Everything +# else in CFLAGS, the Debian hardening flags included, still applies to them. +PRELOAD_CFLAGS := $(filter-out -fsanitize=%,$(CFLAGS)) CFLAGS += -I$(top_srcdir)/include/ CFLAGS += $(CFLAGS_GLIB) @@ -81,7 +85,8 @@ include ../lib/common.Makefile daemon-tests-templ-def daemon-tests-templ-def-offer daemon-tests-t38 daemon-tests-evs-dtx \ daemon-tests-transform daemon-tests-http daemon-tests-heuristic daemon-tests-asymmetric \ daemon-tests-dtx-no-shift daemon-tests-rtcp daemon-tests-redis-subscribe daemon-tests-rtp-ext \ - daemon-tests-bundle daemon-tests-dtls \ + daemon-tests-bundle daemon-tests-dtls daemon-tests-rollback \ + daemon-tests-rollback-redis \ daemon-tests-recording daemon-tests-create daemon-tests-alias \ daemon-tests-kernel-api @@ -132,7 +137,7 @@ daemon-tests: daemon-tests-main daemon-tests-jb daemon-tests-pubsub daemon-tests daemon-tests-sdp-orig-replacements daemon-tests-moh daemon-tests-evs-dtx daemon-tests-transform \ daemon-tests-transcode-config daemon-tests-codec-prefs daemon-tests-http daemon-tests-heuristic \ daemon-tests-asymmetric daemon-tests-rtcp daemon-tests-redis-subscribe daemon-tests-rtp-ext \ - daemon-tests-bundle daemon-tests-dtls \ + daemon-tests-bundle daemon-tests-dtls daemon-tests-rollback daemon-tests-rollback-redis \ daemon-tests-recording daemon-tests-create \ daemon-tests-dtx daemon-tests-dtx-cn daemon-tests-dtx-no-shift \ daemon-tests-alias \ @@ -144,6 +149,15 @@ daemon-test-deps: tests-preload.so daemon-tests-main: daemon-test-deps ./auto-test-helper "$@" perl -I../perl auto-daemon-tests.pl +daemon-tests-rollback: daemon-test-deps + ./auto-test-helper "$@" perl -I../perl auto-daemon-tests-rollback.pl + +daemon-tests-rollback-redis: daemon-test-deps + RTPE_REDIS_FORMAT=native ./auto-test-helper "$@-native" \ + perl -I../perl auto-daemon-tests-rollback-redis.pl + RTPE_REDIS_FORMAT=json ./auto-test-helper "$@-json" \ + perl -I../perl auto-daemon-tests-rollback-redis.pl + daemon-tests-jb: daemon-test-deps ./auto-test-helper "$@" perl -I../perl auto-daemon-tests-jb.pl diff --git a/t/auto-daemon-tests-redis-json.pl b/t/auto-daemon-tests-redis-json.pl index eca50a367..1e9ee15e0 100755 --- a/t/auto-daemon-tests-redis-json.pl +++ b/t/auto-daemon-tests-redis-json.pl @@ -65,16 +65,28 @@ sub redis_io { $NGCP::Rtpengine::req_cb = sub { redis_io("*1\r\n\$4\r\nPING\r\n", "+PONG\r\n", "req PING"); redis_i("*5\r\n\$3\r\nSET\r\n\$" . length(cid()) . "\r\n" . cid() . "\r\n\$", "req intro"); - # dumbly expect 4-digit number as length - my $buf; + # the length is however many digits it takes, and a record large enough to + # need more than one read is normal + my $buf = ''; alarm(1); - recv($redis_fd, $buf, 6, 0) or die; + while ($buf !~ /\r\n\z/) { + my $c; + defined(recv($redis_fd, $c, 1, 0)) or die; + $buf .= $c; + } alarm(0); - is(substr($buf, 4, 2), "\r\n", "4-digit number"); + ok($buf =~ /^\d+\r\n\z/, "record length"); my $len = int($buf); - alarm(1); - recv($redis_fd, $buf, $len, 0) or die; + my $rec = ''; + alarm(5); + while (length($rec) < $len) { + my $chunk; + defined(recv($redis_fd, $chunk, $len - length($rec), 0)) or die; + length($chunk) or die "short read from redis socket"; + $rec .= $chunk; + } alarm(0); + $buf = $rec; my $json = decode_json($buf); #print Dumper($json); Test2::Tools::Compare::like($json, $json_exp, "JSON"); diff --git a/t/auto-daemon-tests-redis-subscribe.pl b/t/auto-daemon-tests-redis-subscribe.pl index d650cc459..d5df410b9 100755 --- a/t/auto-daemon-tests-redis-subscribe.pl +++ b/t/auto-daemon-tests-redis-subscribe.pl @@ -159,16 +159,28 @@ sub redis_io { $NGCP::Rtpengine::req_cb = sub { redis_io($redis_fd, "*1\r\n\$4\r\nPING\r\n", "+PONG\r\n", "req PING"); redis_i($redis_fd, "*5\r\n\$3\r\nSET\r\n\$" . length(cid()) . "\r\n" . cid() . "\r\n\$", "req intro"); - # dumbly expect 4-digit number as length - my $buf; + # the length is however many digits it takes, and a record large enough to + # need more than one read is normal + my $buf = ''; alarm(1); - recv($redis_fd, $buf, 6, 0) or die; + while ($buf !~ /\r\n\z/) { + my $c; + defined(recv($redis_fd, $c, 1, 0)) or die; + $buf .= $c; + } alarm(0); - is(substr($buf, 4, 2), "\r\n", "4-digit number"); + ok($buf =~ /^\d+\r\n\z/, "record length"); my $len = int($buf); - alarm(1); - recv($redis_fd, $buf, $len, 0) or die; + my $rec = ''; + alarm(5); + while (length($rec) < $len) { + my $chunk; + defined(recv($redis_fd, $chunk, $len - length($rec), 0)) or die; + length($chunk) or die "short read from redis socket"; + $rec .= $chunk; + } alarm(0); + $buf = $rec; my $json = Bencode::bdecode($buf, 1); #print Dumper($json); Test2::Tools::Compare::like($json, $json_exp, "JSON"); diff --git a/t/auto-daemon-tests-redis.pl b/t/auto-daemon-tests-redis.pl index 071947708..6e446d9ce 100755 --- a/t/auto-daemon-tests-redis.pl +++ b/t/auto-daemon-tests-redis.pl @@ -64,16 +64,28 @@ sub redis_io { $NGCP::Rtpengine::req_cb = sub { redis_io("*1\r\n\$4\r\nPING\r\n", "+PONG\r\n", "req PING"); redis_i("*5\r\n\$3\r\nSET\r\n\$" . length(cid()) . "\r\n" . cid() . "\r\n\$", "req intro"); - # dumbly expect 4-digit number as length - my $buf; + # the length is however many digits it takes, and a record large enough to + # need more than one read is normal + my $buf = ''; alarm(1); - recv($redis_fd, $buf, 6, 0) or die; + while ($buf !~ /\r\n\z/) { + my $c; + defined(recv($redis_fd, $c, 1, 0)) or die; + $buf .= $c; + } alarm(0); - is(substr($buf, 4, 2), "\r\n", "4-digit number"); + ok($buf =~ /^\d+\r\n\z/, "record length"); my $len = int($buf); - alarm(1); - recv($redis_fd, $buf, $len, 0) or die; + my $rec = ''; + alarm(5); + while (length($rec) < $len) { + my $chunk; + defined(recv($redis_fd, $chunk, $len - length($rec), 0)) or die; + length($chunk) or die "short read from redis socket"; + $rec .= $chunk; + } alarm(0); + $buf = $rec; my $json = Bencode::bdecode($buf, 1); #print Dumper($json); Test2::Tools::Compare::like($json, $json_exp, "JSON"); diff --git a/t/auto-daemon-tests-rollback-redis.pl b/t/auto-daemon-tests-rollback-redis.pl new file mode 100644 index 000000000..c6d7a3540 --- /dev/null +++ b/t/auto-daemon-tests-rollback-redis.pl @@ -0,0 +1,553 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use Bencode; +use JSON; +use NGCP::Rtpengine::AutoTest; +use File::Temp (); +use IO::Select; +use POSIX (); +use Socket qw(AF_INET SOCK_STREAM sockaddr_in inet_aton); +use Test::More; + +my $redis_format = $ENV{RTPE_REDIS_FORMAT} // 'json'; + +# Fake Redis server, run in a forked child. +# +# The daemon writes to Redis from the poller thread as well as after a signalling +# request, so a test servicing it only between requests eventually leaves a write +# unanswered, which blocks the daemon in redis_consume(). A separate process +# always answers; the parent reads what was stored through the files below. +# +# $state/seq number of SETs the child has serviced +# $state/last value of the most recent SET +# $state/override if present, served in place of the next GET, then removed + +my $redis_listener; +socket($redis_listener, AF_INET, SOCK_STREAM, 0) or die; +bind($redis_listener, sockaddr_in(6379, inet_aton('203.0.113.42'))) or die; +listen($redis_listener, 10) or die; + +my $state = File::Temp::tempdir("rollback-redis-XXXXXX", TMPDIR => 1, CLEANUP => 1); +write_file("$state/seq", "0"); + +sub write_file { + my ($path, $content) = @_; + open(my $fh, '>', "$path.tmp") or die "$path: $!"; + binmode($fh); + print $fh $content; + close($fh) or die; + rename("$path.tmp", $path) or die "$path: $!"; +} + +sub read_file { + my ($path) = @_; + open(my $fh, '<', $path) or return undef; + binmode($fh); + local $/; + my $content = <$fh>; + close($fh); + return $content; +} + +sub server_read_exact { + my ($fd, $len) = @_; + my $buf = ''; + while (length($buf) < $len) { + my $part; + defined(recv($fd, $part, $len - length($buf), 0)) or return undef; + length($part) or return undef; + $buf .= $part; + } + return $buf; +} + +sub server_read_line { + my ($fd) = @_; + my $buf = ''; + while ($buf !~ /\r\n\z/) { + my $byte = server_read_exact($fd, 1); + defined($byte) or return undef; + $buf .= $byte; + } + $buf =~ s/\r\n\z//; + return $buf; +} + +sub server_read_command { + my ($fd) = @_; + my $intro = server_read_line($fd); + defined($intro) && $intro =~ /^\*(\d+)\z/ or return undef; + my @args; + for (1 .. $1) { + my $bulk = server_read_line($fd); + defined($bulk) && $bulk =~ /^\$(\d+)\z/ or return undef; + my $arg = server_read_exact($fd, $1); + defined($arg) or return undef; + server_read_exact($fd, 2); + push @args, $arg; + } + return \@args; +} + +# Answer every command the daemon can send, so it is never left waiting. +sub redis_server { + my %store; + my $sets = 0; + my $select = IO::Select->new($redis_listener); + + while (1) { + for my $fh ($select->can_read(1)) { + if ($fh == $redis_listener) { + my $client; + accept($client, $redis_listener) or next; + $select->add($client); + next; + } + my $command = server_read_command($fh); + if (!$command) { + $select->remove($fh); + close($fh); + next; + } + my $verb = uc($command->[0]); + if ($verb eq 'PING') { + send($fh, "+PONG\r\n", 0); + } + elsif ($verb eq 'INFO') { + my $info = "role:master\r\n"; + send($fh, '$' . length($info) . "\r\n$info\r\n", 0); + } + elsif ($verb eq 'TYPE') { + send($fh, "+none\r\n", 0); + } + elsif ($verb eq 'KEYS') { + my @keys = keys %store; + my $reply = '*' . scalar(@keys) . "\r\n"; + $reply .= '$' . length($_) . "\r\n$_\r\n" for @keys; + send($fh, $reply, 0); + } + elsif ($verb eq 'GET') { + my $value = read_file("$state/override"); + if (defined $value) { + unlink("$state/override"); + } + else { + $value = $store{$command->[1]}; + } + if (defined $value) { + send($fh, '$' . length($value) . "\r\n$value\r\n", 0); + } + else { + send($fh, "\$-1\r\n", 0); + } + } + elsif ($verb eq 'SET') { + $store{$command->[1]} = $command->[2]; + write_file("$state/last", $command->[2]); + write_file("$state/seq", ++$sets); + send($fh, "+OK\r\n", 0); + } + elsif ($verb eq 'DEL') { + delete $store{$command->[1]}; + send($fh, ":1\r\n", 0); + } + else { + send($fh, "+OK\r\n", 0); + } + } + } +} + +my $redis_pid = fork(); +defined($redis_pid) or die "cannot fork Redis server"; +if (!$redis_pid) { + $SIG{TERM} = sub { POSIX::_exit(0) }; + redis_server(); + POSIX::_exit(0); +} +# The listener stays open here: under the preload's fake network, closing it in +# the parent removes the socket the child is accepting on. +END { kill('TERM', $redis_pid) if $redis_pid } + +sub redis_sets_seen { + return int(read_file("$state/seq") // 0); +} + +# The daemon writes to Redis before it answers, so the record is normally there +# already; poll briefly to cover the child not having flushed it yet. +sub redis_record_after { + my ($before) = @_; + for (1 .. 500) { + return read_file("$state/last") if redis_sets_seen() > $before; + select(undef, undef, undef, 0.01); + } + die 'no Redis update seen after the daemon answered'; +} + +sub serve_next_get { + my ($record) = @_; + write_file("$state/override", $record); +} + +my ($expected_pending, $last_record); + +sub decode_record { + my ($record) = @_; + return $redis_format eq 'json' ? decode_json($record) : Bencode::bdecode($record, 1); +} + +sub encode_record { + my ($record) = @_; + return encode_json($record) if $redis_format eq 'json'; + my $as_strings; + $as_strings = sub { + my ($value) = @_; + return { map { $_ => $as_strings->($value->{$_}) } keys %$value } + if ref($value) eq 'HASH'; + return [ map { $as_strings->($_) } @$value ] if ref($value) eq 'ARRAY'; + my $copy = $value; + return \$copy; + }; + return Bencode::bencode($as_strings->($record)); +} + +# Record values are escaped by the encoder; JSON records carry them percent +# encoded, native (bencode) records do not. +sub field { + my ($value) = @_; + return undef if !defined $value; + $value =~ s/%([0-9a-fA-F]{2})/chr(hex($1))/ge if $redis_format eq 'json'; + return $value; +} + +# A field that is not a number must disable rollback for the call without +# discarding the call itself. +sub checkpoint_with_invalid_field_type { + my ($record) = @_; + my $decoded = decode_record($record); + $decoded->{'checkpoint-0'}{pending} = 'banana'; + return encode_record($decoded); +} + +sub inspect_checkpoint { + my ($record) = @_; + my $decoded = decode_record($record); + my $checkpoint = $decoded->{'checkpoint-0'}; + ok(defined $checkpoint, "$redis_format record contains a checkpoint"); + # keyed on the monologue id, so both sides of the dialogue carry one + ok(defined $decoded->{'checkpoint-1'}, + "$redis_format both monologues carry a checkpoint"); + is(field($checkpoint->{pending}) ? 1 : 0, $expected_pending, + "$redis_format pending state serialized"); + # The snapshot is a nested call record, so it is checked for presence rather + # than shape: what it must contain is asserted by restoring from it. + ok(!$expected_pending || length(field($checkpoint->{snapshot}) // ''), + "$redis_format pending checkpoint carries a snapshot"); + # A snapshot never leaves the daemon, so it is bencode whatever the record is. + ok(!$expected_pending || (field($checkpoint->{snapshot}) // '') =~ /^d/, + "$redis_format snapshot is bencode"); +} + +# What rollback restored, judged against the record rather than against `query`. +# +# query exposes only a subset of a media's state, so a rollback can stop restoring +# the rest with every existing assertion still green -- which happened. The record +# carries all of it: the committed state must serialise the same before and after. +sub durable_state { + my ($record) = @_; + return durable_fields(decode_record($record)); +} + +# A snapshot is a call record too, so it is filtered the same way. It is always +# bencode, whatever the record around it is. +sub checkpoint_snapshot { + my ($record) = @_; + my $checkpoint = decode_record($record)->{'checkpoint-0'} or return undef; + my $snapshot = field($checkpoint->{snapshot}) or return undef; + return durable_fields(Bencode::bdecode($snapshot, 1)); +} + +sub durable_fields { + my ($decoded) = @_; + my %out; + for my $key (keys %$decoded) { + # Checkpoint entries describe the pending exchange, not the committed + # state, and are expected to differ. + next if $key =~ /^checkpoint/; + # Sockets and endpoint maps are a per-call pool. A rejected offer can add + # to it, and rollback repoints the media rather than freeing what was + # allocated, exactly as the feature has always behaved. What must match is + # the dialogue's own state, not the size of the pool behind it. + next if $key =~ /^(?:sfd|map|map_sfds|maps)-\d+$/; + if ($key eq 'json') { + my %call = %{$decoded->{$key}}; + # Wall-clock and bookkeeping that moves on its own. + delete @call{qw(created created_us created_ts last_signal deleted + ml_deleted last_redis_update num_sfds num_maps)}; + $out{$key} = \%call; + next; + } + $out{$key} = $decoded->{$key}; + } + return \%out; +} + +sub redis_rtpe_req { + my ($pending, @request) = @_; + my $before = redis_sets_seen(); + my $response = rtpe_req(@request); + $last_record = redis_record_after($before); + $expected_pending = $pending; + inspect_checkpoint($last_record); + return $response; +} + +sub assert_record_without_checkpoint { + my ($record) = @_; + my $decoded = decode_record($record); + my @checkpoints = grep { /^checkpoint-/ } keys %$decoded; + is(scalar(@checkpoints), 0, + "$redis_format invalid checkpoint is omitted from the restored call record"); +} + +sub sdp { + my ($address, $port, $ufrag, $pwd, $key, $direction) = @_; + return "v=0\r\no=- 2 2 IN IP4 $address\r\ns=rollback-redis-secure\r\n" + . "c=IN IP4 $address\r\nt=0 0\r\nm=audio $port RTP/SAVP 0\r\n" + . "a=rtpmap:0 PCMU/8000\r\na=$direction\r\n" + . "a=ice-ufrag:$ufrag\r\na=ice-pwd:$pwd\r\n" + . "a=candidate:1 1 UDP 2130706431 $address $port typ host\r\n" + . "a=crypto:1 AES_CM_128_HMAC_SHA1_80 inline:$key\r\n"; +} + +# Plain RTP, and the same media upgraded to DTLS-SRTP. Used to check that a +# rollback removes state the rejected offer introduced, rather than only +# overwriting state that existed in both. +sub plain_sdp { + my ($address, $port) = @_; + return "v=0\r\no=- 4 4 IN IP4 $address\r\ns=rollback-upgrade\r\n" + . "c=IN IP4 $address\r\nt=0 0\r\nm=audio $port RTP/AVP 0\r\n" + . "a=rtpmap:0 PCMU/8000\r\na=sendrecv\r\n"; +} + +sub dtls_upgrade_sdp { + my ($address, $port) = @_; + return "v=0\r\no=- 4 5 IN IP4 $address\r\ns=rollback-upgrade\r\n" + . "c=IN IP4 $address\r\nt=0 0\r\nm=audio $port UDP/TLS/RTP/SAVP 0\r\n" + . "a=rtpmap:0 PCMU/8000\r\na=sendrecv\r\na=setup:actpass\r\n" + . "a=fingerprint:sha-256 " . join(':', ('AB') x 32) . "\r\n" + . "a=tls-id:upgradetlsid0123456789abcdef\r\n" + # ICE too, so the rollback has candidates to remove as well as DTLS state. + . "a=ice-ufrag:upgradeUfrag\r\na=ice-pwd:upgradePassword01234567\r\n" + . "a=candidate:1 1 UDP 2130706431 $address $port typ host\r\n"; +} + +sub secure_parameters { + my ($body) = @_; + my @parameters = $body =~ /^(a=(?:ice-ufrag|ice-pwd):.*|a=crypto:1 .*)$/mg; + return \@parameters; +} + +# A second interface so a rejected offer can move the media somewhere else. +my @daemon_args = (qw(--config-file=none -t -1 -i foo/203.0.113.1 -i bar/203.0.113.2 + -n 2233 -c 12346 -f -L 7 -E --redis-num-threads=1), + "--redis=203.0.113.42:6379/15", "--redis-format=$redis_format"); +$NGCP::Rtpengine::AutoTest::port = 2233; +autotest_start(@daemon_args) or die; + +new_call; +my ($call_id, $from_tag, $to_tag) = (cid(), ft(), tt()); +my $via_branch = 'rollback-redis-branch'; +my $response = redis_rtpe_req(1, 'offer', 'tracked Redis offer', { + 'from-tag' => $from_tag, 'via-branch' => $via_branch, flags => ['track-state'], + sdp => sdp('198.51.100.80', 12000, 'oldRedisUfrag', + 'oldRedisPassword012345678', 'MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkw', 'sendrecv'), +}); +my $secure_parameters = secure_parameters($response->{sdp}); + +redis_rtpe_req(0, 'answer', 'tracked Redis answer', { + 'from-tag' => $from_tag, 'to-tag' => $to_tag, 'via-branch' => $via_branch, + sdp => sdp('198.51.100.81', 12010, 'answerRedisUfrag', + 'answerRedisPassword012345', 'QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVo5ODc2', 'sendrecv'), +}); +$response = redis_rtpe_req(1, 'offer', 'offer later rejected by the far end', { + 'from-tag' => $from_tag, 'to-tag' => $to_tag, 'via-branch' => $via_branch, + sdp => sdp('198.51.100.82', 12020, 'newRedisUfrag', + 'newRedisPassword012345678', 'YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXowMTIz', 'sendonly'), +}); +my $pending_record = $last_record; + +NGCP::Rtpengine::AutoTest::shut_rtpe(); +autotest_start(@daemon_args) or die; + +$response = redis_rtpe_req(0, 'rollback', 'rollback after Redis takeover', { + 'call-id' => $call_id, 'from-tag' => $from_tag, 'to-tag' => $to_tag, + 'via-branch' => $via_branch, +}); +is($response->{'rolled-back'}, 1, 'pending checkpoint survives Redis takeover'); + +my $query = rtpe_req('query', 'query rolled-back Redis call', {'call-id' => $call_id}); +ok($query->{tags}{$from_tag}{medias}[0]{streams}[0]{'local port'}, + 'selected media socket survives takeover rollback'); +is($query->{tags}{$from_tag}{medias}[0]{streams}[0]{endpoint}{address}, '198.51.100.80', + 'remote media endpoint survives takeover rollback'); + +$response = redis_rtpe_req(1, 'offer', 'verify restored Redis media state', { + 'call-id' => $call_id, 'from-tag' => $from_tag, 'to-tag' => $to_tag, + 'via-branch' => $via_branch, + sdp => sdp('198.51.100.80', 12000, 'oldRedisUfrag', + 'oldRedisPassword012345678', 'MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkw', 'sendrecv'), +}); +is_deeply(secure_parameters($response->{sdp}), $secure_parameters, + 'ICE credentials and committed SDES key survive takeover rollback'); + +redis_rtpe_req(0, 'answer', 'commit exchange after takeover rollback', { + 'call-id' => $call_id, 'from-tag' => $from_tag, 'to-tag' => $to_tag, + 'via-branch' => $via_branch, + sdp => sdp('198.51.100.81', 12010, 'answerRedisUfrag', + 'answerRedisPassword012345', 'QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVo5ODc2', 'sendrecv'), +}); + +NGCP::Rtpengine::AutoTest::shut_rtpe(); +autotest_start(@daemon_args) or die; +$response = redis_rtpe_req(0, 'rollback', 'rollback after committed state crosses Redis', { + 'call-id' => $call_id, 'from-tag' => $from_tag, 'to-tag' => $to_tag, + 'via-branch' => $via_branch, +}); +is($response->{'rolled-back'}, 0, 'committed checkpoint remains consumed in Redis'); + +NGCP::Rtpengine::AutoTest::shut_rtpe(); +serve_next_get(checkpoint_with_invalid_field_type($pending_record)); +autotest_start(@daemon_args) or die; +$query = rtpe_req('query', 'query call restored without invalid checkpoint', { + 'call-id' => $call_id, +}); +ok($query->{tags}{$from_tag}, 'invalid checkpoint data does not discard the restored call'); +my $before_invalid = redis_sets_seen(); +$response = rtpe_req('rollback', 'invalid checkpoint degrades to no rollback state', { + 'call-id' => $call_id, 'from-tag' => $from_tag, 'to-tag' => $to_tag, + 'via-branch' => $via_branch, +}); +assert_record_without_checkpoint(redis_record_after($before_invalid)); +is($response->{'rolled-back'}, 0, 'type-invalid checkpoint is discarded atomically'); + +# --- rollback restores the whole committed state, not just what query shows --- +new_call; +my ($cov_call, $cov_from, $cov_to) = (cid(), ft(), tt()); +redis_rtpe_req(1, 'offer', 'coverage offer', { + 'from-tag' => $cov_from, flags => ['track-state'], direction => [qw(foo foo)], + sdp => sdp('198.51.100.90', 14000, 'covUfrag', 'covPassword0123456789', + 'MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkw', 'sendrecv'), +}); +redis_rtpe_req(0, 'answer', 'coverage answer', { + 'from-tag' => $cov_from, 'to-tag' => $cov_to, + sdp => sdp('198.51.100.91', 14010, 'covAnsUfrag', 'covAnsPassword012345', + 'QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVo5ODc2', 'sendrecv'), +}); +my $committed_record = $last_record; + +# A rejected offer that moves the media to the other interface. +redis_rtpe_req(1, 'offer', 'coverage rejected offer', { + 'from-tag' => $cov_from, 'to-tag' => $cov_to, direction => [qw(bar bar)], + sdp => sdp('198.51.100.92', 14020, 'covNewUfrag', 'covNewPassword012345', + 'YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXowMTIz', 'sendonly'), +}); +isnt(durable_state($last_record), durable_state($committed_record), + 'the rejected offer really changed the stored state'); + +my $before_roll = redis_sets_seen(); +my $cov_rollback = rtpe_req('rollback', 'coverage rollback', { + 'call-id' => $cov_call, 'from-tag' => $cov_from, 'to-tag' => $cov_to, +}); +is($cov_rollback->{'rolled-back'}, 1, 'coverage rollback applied'); +is_deeply(durable_state(redis_record_after($before_roll)), + durable_state($committed_record), + 'rollback restores the committed state in full'); + +# --- a rollback must remove state the rejected offer introduced --- +# +# Fields the encoder writes only when set -- tls_id, the DTLS fingerprint, the +# preferred hash function, the endpoint map -- are absent from a snapshot taken +# before they existed. Restoring has to treat that absence as "clear this", not +# as "leave it alone", or an upgrade to DTLS survives its own rollback. +new_call; +my ($up_call, $up_from, $up_to) = (cid(), ft(), tt()); +redis_rtpe_req(1, 'offer', 'upgrade: plain offer', { + 'from-tag' => $up_from, flags => ['track-state'], + sdp => plain_sdp('198.51.100.95', 15000), +}); +redis_rtpe_req(0, 'answer', 'upgrade: plain answer', { + 'from-tag' => $up_from, 'to-tag' => $up_to, + sdp => plain_sdp('198.51.100.96', 15010), +}); +my $plain_record = $last_record; + +redis_rtpe_req(1, 'offer', 'upgrade: rejected DTLS offer', { + 'from-tag' => $up_from, 'to-tag' => $up_to, + sdp => dtls_upgrade_sdp('198.51.100.97', 15020), +}); +my $upgraded = decode_record($last_record); +ok(defined field($upgraded->{'media-0'}{hash_func}), + 'the rejected offer really introduced DTLS state'); + +my $before_up = redis_sets_seen(); +my $up_rollback = rtpe_req('rollback', 'upgrade: rollback', { + 'call-id' => $up_call, 'from-tag' => $up_from, 'to-tag' => $up_to, +}); +is($up_rollback->{'rolled-back'}, 1, 'upgrade rollback applied'); +is_deeply(durable_state(redis_record_after($before_up)), + durable_state($plain_record), + 'rollback removes DTLS state the rejected offer introduced'); + +# --- the snapshot-only state has to survive a rollback too --- +# +# ICE credentials and candidates, endpoint learning, offered codecs, tls_id, the +# preferred hash function and the endpoint map are written into snapshots only, +# so a record comparison cannot see them: a rollback could stop restoring any of +# them with every assertion above still green. Comparing the snapshot taken +# before the rejected offer against one taken after the rollback covers all of +# them at once, because a snapshot is taken before its offer is applied and so +# describes the state the rollback was supposed to reproduce. +new_call; +my ($rt_call, $rt_from, $rt_to) = (cid(), ft(), tt()); +redis_rtpe_req(1, 'offer', 'round trip: offer', { + 'from-tag' => $rt_from, flags => ['track-state'], + sdp => sdp('198.51.100.98', 15030, 'roundTripUfrag', 'roundTripPassword0123456', + 'Ai0RVBUpx3FYuJEyv1oOTQVHrfXEIQGRxWLXQBvR', 'sendrecv'), +}); +redis_rtpe_req(0, 'answer', 'round trip: answer', { + 'from-tag' => $rt_from, 'to-tag' => $rt_to, + sdp => sdp('198.51.100.99', 15040, 'roundTripAnswer', 'roundTripAnswerPwd012345', + 'HHf1TXWnpZlfXHBw5Q3xTNTIhFvbEHIYnmSMDGqR', 'sendrecv'), +}); + +redis_rtpe_req(1, 'offer', 'round trip: rejected offer', { + 'from-tag' => $rt_from, 'to-tag' => $rt_to, + sdp => sdp('198.51.100.100', 15050, 'roundTripReject', 'roundTripRejectPwd01234', + 'GHi1TXWnpZlfXHBw5Q3xTNTIhFvbEHIYnmSMDGqQ', 'sendrecv'), +}); +my $snapshot_before = checkpoint_snapshot($last_record); +ok($snapshot_before, 'round trip: committed snapshot captured'); + +my $rt_rollback = rtpe_req('rollback', 'round trip: rollback', { + 'call-id' => $rt_call, 'from-tag' => $rt_from, 'to-tag' => $rt_to, +}); +is($rt_rollback->{'rolled-back'}, 1, 'round trip: rollback applied'); + +# The snapshot for this offer is taken before it is applied, so it describes the +# state the rollback restored. +redis_rtpe_req(1, 'offer', 'round trip: offer after rollback', { + 'from-tag' => $rt_from, 'to-tag' => $rt_to, + sdp => sdp('198.51.100.101', 15060, 'roundTripAfter', 'roundTripAfterPwd012345', + 'JKl1TXWnpZlfXHBw5Q3xTNTIhFvbEHIYnmSMDGqP', 'sendrecv'), +}); +my $snapshot_after = checkpoint_snapshot($last_record); +ok($snapshot_after, 'round trip: post-rollback snapshot captured'); + +is_deeply($snapshot_after, $snapshot_before, + 'rollback restores the snapshot-only state as well'); + +NGCP::Rtpengine::AutoTest::shut_rtpe(); +done_testing; diff --git a/t/auto-daemon-tests-rollback.pl b/t/auto-daemon-tests-rollback.pl new file mode 100644 index 000000000..0e5bffe87 --- /dev/null +++ b/t/auto-daemon-tests-rollback.pl @@ -0,0 +1,486 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use NGCP::Rtpengine::Test; +use NGCP::Rtpengine::AutoTest; +use NGCP::Rtpclient::ICE; +use NGCP::Rtpclient::DTLS; +use IO::Multiplex; +use Socket qw(MSG_DONTWAIT); +use Test::More; + +autotest_start(qw(--config-file=none -t -1 -i 203.0.113.1 + -n 2223 -c 12345 -f -L 7 -E -u 2222)) or die; + +sub sdp { + my ($address, $port, $payload, $direction, $extra_media) = @_; + my $codec = $payload == 8 ? 'PCMA' : 'PCMU'; + my $ret = "v=0\r\no=- 1 1 IN IP4 $address\r\ns=rollback\r\n" + . "c=IN IP4 $address\r\nt=0 0\r\nm=audio $port RTP/AVP $payload\r\n" + . "a=rtpmap:$payload $codec/8000\r\na=$direction\r\n"; + $ret .= "m=video " . ($port + 2) . " RTP/AVP 96\r\n" + . "a=rtpmap:96 VP8/90000\r\na=$direction\r\n" if $extra_media; + return $ret; +} + +sub rollback { + my ($via_branch) = @_; + my %req = ('call-id' => cid(), 'from-tag' => ft(), 'to-tag' => tt()); + $req{'via-branch'} = $via_branch if defined $via_branch; + return rtpe_req('rollback', 'rollback state', \%req); +} + +sub negotiated_tags { + my ($state) = @_; + # __fill_stream() refreshes ps->last_packet_us on each offer. Query exposes + # that value, truncated to seconds, as both "last packet" and "last user + # packet". It is liveness state rather than negotiated media state. + # Whole-tag comparisons are safe only while no media has flowed: once packets + # arrive, the per-media SSRC lists also contain traffic-derived statistics. + # Such tests must compare only the negotiated fields they intend to restore. + for my $tag (values %{$state->{tags}}) { + for my $media (@{$tag->{medias}}) { + for my $stream (@{$media->{streams}}) { + delete $stream->{'last packet'}; + delete $stream->{'last user packet'}; + } + } + } + return $state->{tags}; +} + +sub secure_sdp { + my ($address, $port, $ufrag, $pwd, $key, $direction) = @_; + return "v=0\r\no=- 2 2 IN IP4 $address\r\ns=rollback-secure\r\n" + . "c=IN IP4 $address\r\nt=0 0\r\nm=audio $port RTP/SAVP 0\r\n" + . "a=rtpmap:0 PCMU/8000\r\na=$direction\r\n" + . "a=ice-ufrag:$ufrag\r\na=ice-pwd:$pwd\r\n" + . "a=candidate:1 1 UDP 2130706431 $address $port typ host\r\n" + . "a=candidate:1 2 UDP 2130706430 $address " . ($port + 1) . " typ host\r\n" + . "a=crypto:1 AES_CM_128_HMAC_SHA1_80 inline:$key\r\n"; +} + +sub dtls_sdp { + my ($address, $port, $fingerprint, $tls_id, $setup) = @_; + return "v=0\r\no=- 3 3 IN IP4 $address\r\ns=rollback-dtls\r\n" + . "c=IN IP4 $address\r\nt=0 0\r\nm=audio $port UDP/TLS/RTP/SAVP 0\r\n" + . "a=rtpmap:0 PCMU/8000\r\na=setup:$setup\r\n" + . "a=fingerprint:sha-256 $fingerprint\r\na=tls-id:$tls_id\r\n"; +} + +sub secure_parameters { + my ($sdp) = @_; + my @parameters = $sdp =~ /^(a=(?:ice-ufrag|ice-pwd):.*|a=crypto:1 .*)$/mg; + return \@parameters; +} + +my ($rollback_dtls, $rollback_dtls_mux, $rollback_dtls_connected, + @rollback_dtls_components); +my $rollback_dtls_output = sub { + my ($component, $data) = @_; + my ($socket, $port) = @{$rollback_dtls_components[$component]}; + snd($socket, $port, $data); +}; + +sub mux_input { + my ($self, $mux, $fh, $input) = @_; + my $peer = $mux->udp_peer($fh); + $rollback_dtls->input($fh, $input, $peer); + for my $component (@$rollback_dtls) { + return unless $component->{_connected}; + } + return if $rollback_dtls_connected; + $rollback_dtls_connected = 1; + pass('DTLS re-handshake succeeds with restored configuration'); + $mux->endloop(); +} + +sub consecutive_offer_rollback { + my ($label) = @_; + new_call; + rtpe_req('offer', "$label initial offer", { + 'from-tag' => ft(), flags => ['track-state'], + sdp => sdp('198.51.100.60', 10000, 0, 'sendrecv'), + }); + rtpe_req('answer', "$label initial answer", { + 'from-tag' => ft(), 'to-tag' => tt(), + sdp => sdp('198.51.100.61', 11000, 0, 'sendrecv'), + }); + my $committed_state = rtpe_req('query', "$label committed state", {}); + + my $first = rtpe_req('offer', "$label first pending offer", { + 'from-tag' => ft(), 'to-tag' => tt(), + sdp => sdp('198.51.100.62', 10020, 8, 'sendonly'), + }); + my $second = rtpe_req('offer', "$label second pending offer", { + 'from-tag' => ft(), 'to-tag' => tt(), + sdp => sdp('198.51.100.63', 10030, 8, 'recvonly', 1), + }); + my $rolled_back = rollback(); + is($rolled_back->{'rolled-back'}, 1, "$label rollback consumes the original snapshot"); + my $restored_state = rtpe_req('query', "$label restored state", {}); + is_deeply(negotiated_tags($restored_state), negotiated_tags($committed_state), + "$label rollback restores the originally committed state"); +} + +new_call; +my $resp = rtpe_req('offer', 'tracked initial offer', { + 'from-tag' => ft(), flags => ['track-state'], supports => ['rollback'], + sdp => sdp('198.51.100.10', 4000, 0, 'sendrecv'), +}); +is_deeply($resp->{supported}, ['rollback'], 'rollback capability advertised'); + +$resp = rtpe_req('answer', 'tracked initial answer', { + 'from-tag' => ft(), 'to-tag' => tt(), + sdp => sdp('198.51.100.20', 5000, 0, 'sendrecv'), +}); +my $committed = rtpe_req('query', 'query committed state', {}); + +$resp = rtpe_req('offer', 'failed renegotiation', { + 'from-tag' => ft(), 'to-tag' => tt(), + sdp => sdp('198.51.100.11', 4010, 8, 'sendonly', 1), +}); +$resp = rollback(); +is($resp->{'rolled-back'}, 1, 'pending checkpoint rolls back'); +my $restored = rtpe_req('query', 'query restored state', {}); +is_deeply(negotiated_tags($restored), negotiated_tags($committed), + 'query media state restored'); +$resp = rollback(); +is($resp->{'rolled-back'}, 0, 'repeated rollback is a no-op'); + +rtpe_req('offer', 'completed renegotiation offer', { + 'from-tag' => ft(), 'to-tag' => tt(), + sdp => sdp('198.51.100.12', 4020, 8, 'sendrecv'), +}); +$resp = rtpe_req('answer', 'completed renegotiation answer', { + 'from-tag' => ft(), 'to-tag' => tt(), + sdp => sdp('198.51.100.22', 5020, 8, 'sendrecv'), +}); +$resp = rollback(2); +is($resp->{'rolled-back'}, 0, 'completed exchange cannot be rolled back'); + +consecutive_offer_rollback('consecutive offers'); + +my ($subscription_a, $subscription_b, $subscription_sink) = new_call( + [qw(198.51.100.80 12000)], + [qw(198.51.100.80 12010)], + [qw(198.51.100.80 12020)], +); +my $subscription_offer = rtpe_req('offer', 'subscription rollback initial offer', { + 'from-tag' => ft(), flags => ['track-state'], + sdp => sdp('198.51.100.80', 12000, 0, 'sendrecv'), +}); +my ($subscription_port_a) = $subscription_offer->{sdp} =~ /^m=audio (\d+)/m; +my $subscription_answer = rtpe_req('answer', 'subscription rollback initial answer', { + 'from-tag' => ft(), 'to-tag' => tt(), + sdp => sdp('198.51.100.80', 12010, 0, 'sendrecv'), +}); +my ($subscription_port_b) = $subscription_answer->{sdp} =~ /^m=audio (\d+)/m; +my $subscription = rtpe_req('subscribe request', 'subscription before rollback', { + 'from-tag' => ft(), +}); +my ($subscription_port_sink) = $subscription->{sdp} =~ /^m=audio (\d+)/m; +ok($subscription_port_a && $subscription_port_b && $subscription_port_sink, + 'subscription relay ports captured'); +rtpe_req('subscribe answer', 'subscription before rollback answer', { + 'from-tag' => $subscription->{'from-tag'}, + 'to-tag' => $subscription->{'to-tag'}, + sdp => sdp('198.51.100.80', 12020, 0, 'recvonly'), +}); +snd($subscription_a, $subscription_port_b, + rtp(0, 5000, 8000, 0x7890, "\x55" x 160)); +rcv($subscription_b, $subscription_port_a, + rtpm(0, 5000, 8000, 0x7890, "\x55" x 160)); +rcv($subscription_sink, $subscription_port_sink, + rtpm(0, 5000, 8000, 0x7890, "\x55" x 160)); +my $subscription_committed = rtpe_req('query', 'subscription committed state', {}); +my $subscription_pending = rtpe_req('offer', 'subscription rejected renegotiation', { + 'from-tag' => ft(), 'to-tag' => tt(), + sdp => sdp('198.51.100.81', 12030, 8, 'sendonly'), +}); +my $subscription_rollback = rollback(); +is($subscription_rollback->{'rolled-back'}, 1, + 'subscription call rolls back rejected renegotiation'); +my $subscription_restored = rtpe_req('query', 'subscription restored state', {}); +# Do not compare the complete query: restoring the committed remote endpoint is +# itself an endpoint change and therefore triggers call_stream_crypto_reset(), +# which intentionally resets the SSRC's ext_seq along with the crypto context. +for my $tag (keys %{$subscription_committed->{tags}}) { + is_deeply($subscription_restored->{tags}{$tag}{subscriptions}, + $subscription_committed->{tags}{$tag}{subscriptions}, + "rollback preserves subscriptions for $tag"); + is_deeply($subscription_restored->{tags}{$tag}{subscribers}, + $subscription_committed->{tags}{$tag}{subscribers}, + "rollback preserves subscribers for $tag"); +} +is_deeply($subscription_restored->{tags}{ft()}{medias}[0]{streams}[0]{endpoint}, + $subscription_committed->{tags}{ft()}{medias}[0]{streams}[0]{endpoint}, + 'active subscription resolves to the restored source endpoint'); +snd($subscription_a, $subscription_port_b, + rtp(0, 5001, 8160, 0x7890, "\x66" x 160)); +rcv($subscription_b, $subscription_port_a, + rtpm(0, 5001, 8160, 0x7890, "\x66" x 160)); +rcv($subscription_sink, $subscription_port_sink, + rtpm(0, 5001, 8160, 0x7890, "\x66" x 160)); + +new_call; +rtpe_req('offer', 'untracked offer', { + 'from-tag' => ft(), sdp => sdp('198.51.100.30', 6000, 0, 'sendrecv'), +}); +rtpe_req('answer', 'untracked answer', { + 'from-tag' => ft(), 'to-tag' => tt(), + sdp => sdp('198.51.100.40', 7000, 0, 'sendrecv'), +}); +$resp = rollback(); +is($resp->{'rolled-back'}, 0, 'untracked call has no checkpoint'); + +my ($secure_sock) = new_call([qw(198.51.100.50 8000)]); +my $secure_offer = secure_sdp('198.51.100.50', 8000, 'oldUfrag', + 'oldPassword0123456789012', 'MTIzNDU2Nzg5MDEyMzQ1Njc4OTAxMjM0NTY3ODkw', 'sendrecv'); +$resp = rtpe_req('offer', 'tracked ICE and SDES offer', { + 'from-tag' => ft(), 'via-branch' => 'rollback-branch', + flags => ['track-state'], sdp => $secure_offer, +}); +my $secure_parameters = secure_parameters($resp->{sdp}); +rtpe_req('answer', 'tracked ICE and SDES answer', { + 'from-tag' => ft(), 'to-tag' => tt(), + sdp => secure_sdp('198.51.100.51', 9000, 'answerUfrag', + 'answerPassword0123456789', 'QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVo5ODc2', 'sendrecv'), +}); +rtpe_req('offer', 'ICE restart and SDES rekey', { + 'from-tag' => ft(), 'to-tag' => tt(), 'via-branch' => 'rollback-branch', + sdp => secure_sdp('198.51.100.52', 8010, 'newUfrag', + 'newPassword0123456789012', 'YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXowMTIz', 'sendonly'), +}); +my $branch_error = rtpe_raw_req({command => 'rollback', 'call-id' => cid(), + 'from-tag' => ft(), 'to-tag' => tt(), 'via-branch' => 'wrong-branch'}); +like($branch_error, qr/Unknown dialogue/, 'incorrect via-branch does not select the dialogue'); +$resp = rollback(2, 'rollback-branch'); +is($resp->{'rolled-back'}, 1, 'ICE restart and SDES rekey roll back'); +$resp = rtpe_req('offer', 'replay pre-restart ICE and SDES offer', { + 'from-tag' => ft(), 'to-tag' => tt(), 'via-branch' => 'rollback-branch', sdp => $secure_offer, +}); +is_deeply(secure_parameters($resp->{sdp}), $secure_parameters, + 'local ICE credentials and SDES key are restored'); +my ($secure_port) = $resp->{sdp} =~ /^m=audio (\d+)/m; +my ($local_ufrag) = $resp->{sdp} =~ /^a=ice-ufrag:(\S+)/m; +my ($local_pwd) = $resp->{sdp} =~ /^a=ice-pwd:(\S+)/m; +my @restored_check = rcv($secure_sock, -1, + qr/^\x00\x01\x00.\x21\x12\xa4\x42(............)/s); +snd($secure_sock, $secure_port, NGCP::Rtpclient::ICE::stun_succ( + $secure_port, $restored_check[2], 'oldPassword0123456789012')); +while (1) { + my $discard = ''; + last unless defined $secure_sock->recv($discard, 65535, MSG_DONTWAIT); +} +my ($stun_packet) = NGCP::Rtpclient::ICE::stun_req(0, 65527, 1, + 'oldUfrag', $local_ufrag, $local_pwd); +snd($secure_sock, $secure_port, $stun_packet); +rcv($secure_sock, -1, qr/^\x01\x01\x00.\x21\x12\xa4\x42/s); +pass('restored ICE credentials authenticate a connectivity check'); +rollback(2, 'rollback-branch'); + +my ($dtls_sock) = new_call([qw(198.51.100.55 9500)]); +$rollback_dtls_mux = IO::Multiplex->new(); +$rollback_dtls_mux->set_callback_object(__PACKAGE__); +$rollback_dtls = NGCP::Rtpclient::DTLS::Group->new($rollback_dtls_mux, + $rollback_dtls_output, [[$dtls_sock]]); +my $original_fingerprint = $rollback_dtls->[0]->fingerprint(); +my $original_dtls_offer = dtls_sdp('198.51.100.55', 9500, $original_fingerprint, + 'rollback-original', 'passive'); +rtpe_req('offer', 'tracked DTLS offer', { + 'from-tag' => ft(), flags => ['track-state'], SDES => 'off', + sdp => $original_dtls_offer, +}); +my $dtls_answer = rtpe_req('answer', 'tracked DTLS answer', { + 'from-tag' => ft(), 'to-tag' => tt(), SDES => 'off', + sdp => dtls_sdp('198.51.100.56', 9510, join(':', ('BB') x 32), + 'rollback-answer', 'active'), +}); +my ($restored_dtls_port) = $dtls_answer->{sdp} =~ /^m=audio (\d+)/m; +ok($restored_dtls_port, 'committed DTLS relay port captured'); +rtpe_req('offer', 'DTLS fingerprint and role change later rejected', { + 'from-tag' => ft(), 'to-tag' => tt(), SDES => 'off', + sdp => dtls_sdp('198.51.100.55', 9500, join(':', ('AA') x 32), + 'rollback-rejected', 'active'), +}); +my $dtls_rollback = rollback(2); +is($dtls_rollback->{'rolled-back'}, 1, 'DTLS configuration rolls back'); +$rollback_dtls_mux->add($dtls_sock); +@rollback_dtls_components = ([$dtls_sock, $restored_dtls_port]); +$rollback_dtls->accept(); +$rollback_dtls_mux->loop(); +rtpe_req('delete', 'delete DTLS rollback call', { + 'from-tag' => ft(), 'to-tag' => tt(), +}); + +new_call; +my $fork_from = ft(); +my $fork_a = 'fork-a-' . tt(); +my $fork_b = 'fork-b-' . tt(); +rtpe_req('offer', 'fork A initial offer', { + 'from-tag' => $fork_from, 'via-branch' => 'fork-a', flags => ['track-state'], + sdp => sdp('198.51.100.60', 10000, 0, 'sendrecv'), +}); +rtpe_req('answer', 'fork A initial answer', { + 'from-tag' => $fork_from, 'to-tag' => $fork_a, 'via-branch' => 'fork-a', + sdp => sdp('198.51.100.61', 10010, 0, 'sendrecv'), +}); +rtpe_req('offer', 'fork B initial offer', { + 'from-tag' => $fork_from, 'via-branch' => 'fork-b', flags => ['track-state'], + sdp => sdp('198.51.100.62', 10020, 0, 'sendrecv'), +}); +rtpe_req('answer', 'fork B initial answer', { + 'from-tag' => $fork_from, 'to-tag' => $fork_b, 'via-branch' => 'fork-b', + sdp => sdp('198.51.100.63', 10030, 0, 'sendrecv'), +}); +my $fork_a_offer = rtpe_req('offer', 'fork A rejected renegotiation', { + 'from-tag' => $fork_from, 'to-tag' => $fork_a, 'via-branch' => 'fork-a', + sdp => sdp('198.51.100.64', 10040, 8, 'sendonly'), +}); +my $fork_b_offer = rtpe_req('offer', 'fork B rejected renegotiation', { + 'from-tag' => $fork_from, 'to-tag' => $fork_b, 'via-branch' => 'fork-b', + sdp => sdp('198.51.100.65', 10050, 8, 'recvonly'), +}); +my $fork_error = rtpe_raw_req({command => 'rollback', 'call-id' => cid(), + 'from-tag' => $fork_from, 'to-tag' => $fork_b, 'via-branch' => 'fork-a'}); +like($fork_error, qr/Unknown dialogue/, 'branch and to-tag must identify the same fork'); +my $fork_a_rollback = rtpe_req('rollback', 'rollback fork A', { + 'from-tag' => $fork_from, 'to-tag' => $fork_a, 'via-branch' => 'fork-a', +}); +is($fork_a_rollback->{'rolled-back'}, 1, 'fork A rolls back independently'); +my $fork_a_repeat = rtpe_req('rollback', 'repeat rollback fork A', { + 'from-tag' => $fork_from, 'to-tag' => $fork_a, 'via-branch' => 'fork-a', +}); +is($fork_a_repeat->{'rolled-back'}, 0, 'fork A checkpoint was consumed'); +my $fork_committed = rtpe_req('query', 'fork state before second rollback', {}); +my $fork_b_rollback = rtpe_req('rollback', 'rollback fork B', { + 'from-tag' => $fork_from, 'to-tag' => $fork_b, 'via-branch' => 'fork-b', +}); +is($fork_b_rollback->{'rolled-back'}, 1, 'fork B checkpoint remains pending'); +# The caller monologue is shared between branches. Rolling back the second one +# must not reinstate what rolling back the first one undid. +my $fork_after = rtpe_req('query', 'fork state after both rollbacks', {}); +is_deeply($fork_after->{tags}{$fork_from}{medias}, + $fork_committed->{tags}{$fork_from}{medias}, + 'rolling back the second fork leaves the caller alone'); + +new_call; +rtpe_req('offer', 'stress initial offer', { + 'from-tag' => ft(), flags => ['track-state'], + sdp => sdp('198.51.100.70', 11000, 0, 'sendrecv'), +}); +rtpe_req('answer', 'stress initial answer', { + 'from-tag' => ft(), 'to-tag' => tt(), + sdp => sdp('198.51.100.71', 11010, 0, 'sendrecv'), +}); +for my $iteration (1 .. 10) { + rtpe_req('offer', "stress offer $iteration", { + 'from-tag' => ft(), 'to-tag' => tt(), + sdp => sdp('198.51.100.72', 11020 + $iteration * 2, + $iteration % 2 ? 8 : 0, $iteration % 2 ? 'sendonly' : 'recvonly'), + }); + my $rolled_back = rollback(); + is($rolled_back->{'rolled-back'}, 1, "stress rollback $iteration consumes checkpoint"); +} +my $pending_delete = rtpe_req('offer', 'leave checkpoint pending for delete', { + 'from-tag' => ft(), 'to-tag' => tt(), + sdp => sdp('198.51.100.73', 11100, 8, 'sendonly'), +}); +rtpe_req('delete', 'delete call after repeated rollbacks', { + 'from-tag' => ft(), 'to-tag' => tt(), +}); + +my $error = rtpe_raw_req({command => 'rollback', 'call-id' => 'unknown-call', + 'from-tag' => 'from', 'to-tag' => 'to'}); +like($error, qr/Unknown call-id/, 'unknown call is an error'); +$error = rtpe_raw_req({command => 'rollback', 'call-id' => cid(), + 'from-tag' => 'unknown-tag', 'to-tag' => tt()}); +like($error, qr/Unknown dialogue/, 'unknown dialogue is an error'); + +# --- a merged call must not claim to have rolled back --- +# +# call_merge() renumbers every unique id, and a snapshot is keyed on them, so a +# checkpoint taken before the merge no longer describes anything. It is dropped, +# and rollback says so rather than reporting a success that restored nothing. +new_call; +my $mg_cid = cid(); +my $mg_ft = ft(); +my $mg_tt = tt(); +rtpe_req('offer', 'merge: tracked offer', { + 'from-tag' => $mg_ft, flags => ['track-state'], + sdp => sdp('198.51.100.120', 17000, 0, 'sendrecv'), +}); +rtpe_req('answer', 'merge: answer', { + 'from-tag' => $mg_ft, 'to-tag' => $mg_tt, + sdp => sdp('198.51.100.121', 17010, 0, 'sendrecv'), +}); +rtpe_req('offer', 'merge: rejected offer', { + 'from-tag' => $mg_ft, 'to-tag' => $mg_tt, + sdp => sdp('198.51.100.122', 17020, 8, 'sendonly'), +}); + +new_call; +rtpe_req('offer', 'merge: other call offer', { + 'from-tag' => ft(), sdp => sdp('198.51.100.123', 17030, 0, 'sendrecv'), +}); +rtpe_req('answer', 'merge: other call answer', { + 'from-tag' => ft(), 'to-tag' => tt(), + sdp => sdp('198.51.100.124', 17040, 0, 'sendrecv'), +}); + +# the first call listed survives, so the tracked one is the side being renumbered +rtpe_req('mesh', 'merge the two calls', { + flags => [], + calls => [cid(), $mg_cid], + tags => [ + { from => $mg_ft, to => [$mg_tt] }, + { from => $mg_tt, to => [$mg_ft] }, + ], +}); +my $mg_resp = rtpe_raw_req({command => 'rollback', 'call-id' => $mg_cid, + 'from-tag' => $mg_ft, 'to-tag' => $mg_tt}); +ok(ref($mg_resp) ne 'HASH' || !$mg_resp->{'rolled-back'}, + 'a merged call does not report a rollback it did not perform'); + +# --- both sides of a dialogue are checkpointed together --- +# +# A monologue is shared between forked branches, so one side can already hold a +# checkpoint while the other has never been tracked. Both are taken together, or +# a rollback restores half a dialogue and still reports success. +new_call; +my $sym_from = ft(); +my $sym_a = 'sym-a-' . tt(); +my $sym_b = 'sym-b-' . tt(); +rtpe_req('offer', 'symmetry: branch A tracked offer', { + 'from-tag' => $sym_from, 'via-branch' => 'sym-a', flags => ['track-state'], + sdp => sdp('198.51.100.130', 18000, 0, 'sendrecv'), +}); +rtpe_req('answer', 'symmetry: branch A answer', { + 'from-tag' => $sym_from, 'to-tag' => $sym_a, 'via-branch' => 'sym-a', + sdp => sdp('198.51.100.131', 18010, 0, 'sendrecv'), +}); +# branch B never asks for tracking, but shares the caller monologue with A +rtpe_req('offer', 'symmetry: branch B untracked offer', { + 'from-tag' => $sym_from, 'via-branch' => 'sym-b', + sdp => sdp('198.51.100.132', 18020, 0, 'sendrecv'), +}); +rtpe_req('answer', 'symmetry: branch B answer', { + 'from-tag' => $sym_from, 'to-tag' => $sym_b, 'via-branch' => 'sym-b', + sdp => sdp('198.51.100.133', 18030, 0, 'sendrecv'), +}); +my $sym_committed = rtpe_req('query', 'symmetry: committed state', {}); +rtpe_req('offer', 'symmetry: branch B rejected offer', { + 'from-tag' => $sym_from, 'to-tag' => $sym_b, 'via-branch' => 'sym-b', + sdp => sdp('198.51.100.134', 18040, 8, 'sendonly'), +}); +my $sym_rollback = rtpe_req('rollback', 'symmetry: rollback branch B', { + 'from-tag' => $sym_from, 'to-tag' => $sym_b, 'via-branch' => 'sym-b', +}); +my $sym_restored = rtpe_req('query', 'symmetry: restored state', {}); +is_deeply($sym_restored->{tags}{$sym_b}{medias}, + $sym_committed->{tags}{$sym_b}{medias}, + 'the far side is restored, not just the shared caller'); + +done_testing; diff --git a/t/test-stats.c b/t/test-stats.c index ae99d878a..2c563f4da 100644 --- a/t/test-stats.c +++ b/t/test-stats.c @@ -112,6 +112,13 @@ int main(void) { "answers_ps_max 0 150\n" "answers_ps_avg 0 150\n" "answer_count 0 150\n" + "rollback_time_min 0.000000 150\n" + "rollback_time_max 0.000000 150\n" + "rollback_time_avg 0.000000 150\n" + "rollbacks_ps_min 0 150\n" + "rollbacks_ps_max 0 150\n" + "rollbacks_ps_avg 0 150\n" + "rollback_count 0 150\n" "delete_time_min 0.000000 150\n" "delete_time_max 0.000000 150\n" "delete_time_avg 0.000000 150\n" @@ -596,6 +603,14 @@ int main(void) { "0.000000\n" "avganswerdelay\n" "0.000000\n" + "Min/Max/Avg rollback processing delay\n" + "0.000000/0.000000/0.000000 sec\n" + "minrollbackdelay\n" + "0.000000\n" + "maxrollbackdelay\n" + "0.000000\n" + "avgrollbackdelay\n" + "0.000000\n" "Min/Max/Avg delete processing delay\n" "0.000000/0.000000/0.000000 sec\n" "mindeletedelay\n" @@ -860,6 +875,14 @@ int main(void) { "0\n" "avganswerrequestrate\n" "0\n" + "Min/Max/Avg rollback requests per second\n" + "0/0/0 per sec\n" + "minrollbackrequestrate\n" + "0\n" + "maxrollbackrequestrate\n" + "0\n" + "avgrollbackrequestrate\n" + "0\n" "Min/Max/Avg delete requests per second\n" "0/0/0 per sec\n" "mindeleterequestrate\n" @@ -1263,7 +1286,7 @@ int main(void) { "{\n" "proxies\n" "[\n" - " Proxy | Ping | Offer | Answer | Delete | Query | List | StartRec | StopRec | PauseRec | StartFwd | StopFwd | BlkDTMF | UnblkDTMF | BlkMedia | UnblkMedia | PlayMedia | StopMedia | PlayDTMF | Stats | SlnMedia | UnslnMedia | Pub | SubReq | SubAns | Unsub | InjStart | InjStop | Conn | CLI | Trnsfm | Create | CrtAnsw | Mesh \n" + " Proxy | Ping | Offer | Answer | Rollback | Delete | Query | List | StartRec | StopRec | PauseRec | StartFwd | StopFwd | BlkDTMF | UnblkDTMF | BlkMedia | UnblkMedia | PlayMedia | StopMedia | PlayDTMF | Stats | SlnMedia | UnslnMedia | Pub | SubReq | SubAns | Unsub | InjStart | InjStop | Conn | CLI | Trnsfm | Create | CrtAnsw | Mesh \n" "\n" "]\n" "totalpingcount\n" @@ -1272,6 +1295,8 @@ int main(void) { "0\n" "totalanswercount\n" "0\n" + "totalrollbackcount\n" + "0\n" "totaldeletecount\n" "0\n" "totalquerycount\n" @@ -1365,7 +1390,7 @@ int main(void) { "size\n" "16777208\n" "used\n" - "464\n" + "472\n" "}\n" "]\n" "}\n" @@ -1401,6 +1426,13 @@ int main(void) { "answers_ps_max 0 150\n" "answers_ps_avg 0 150\n" "answer_count 0 150\n" + "rollback_time_min 0.000000 150\n" + "rollback_time_max 0.000000 150\n" + "rollback_time_avg 0.000000 150\n" + "rollbacks_ps_min 0 150\n" + "rollbacks_ps_max 0 150\n" + "rollbacks_ps_avg 0 150\n" + "rollback_count 0 150\n" "delete_time_min 0.000000 150\n" "delete_time_max 0.000000 150\n" "delete_time_avg 0.000000 150\n" @@ -1885,6 +1917,14 @@ int main(void) { "0.000000\n" "avganswerdelay\n" "0.000000\n" + "Min/Max/Avg rollback processing delay\n" + "0.000000/0.000000/0.000000 sec\n" + "minrollbackdelay\n" + "0.000000\n" + "maxrollbackdelay\n" + "0.000000\n" + "avgrollbackdelay\n" + "0.000000\n" "Min/Max/Avg delete processing delay\n" "0.000000/0.000000/0.000000 sec\n" "mindeletedelay\n" @@ -2149,6 +2189,14 @@ int main(void) { "0\n" "avganswerrequestrate\n" "0\n" + "Min/Max/Avg rollback requests per second\n" + "0/0/0 per sec\n" + "minrollbackrequestrate\n" + "0\n" + "maxrollbackrequestrate\n" + "0\n" + "avgrollbackrequestrate\n" + "0\n" "Min/Max/Avg delete requests per second\n" "0/0/0 per sec\n" "mindeleterequestrate\n" @@ -2552,7 +2600,7 @@ int main(void) { "{\n" "proxies\n" "[\n" - " Proxy | Ping | Offer | Answer | Delete | Query | List | StartRec | StopRec | PauseRec | StartFwd | StopFwd | BlkDTMF | UnblkDTMF | BlkMedia | UnblkMedia | PlayMedia | StopMedia | PlayDTMF | Stats | SlnMedia | UnslnMedia | Pub | SubReq | SubAns | Unsub | InjStart | InjStop | Conn | CLI | Trnsfm | Create | CrtAnsw | Mesh \n" + " Proxy | Ping | Offer | Answer | Rollback | Delete | Query | List | StartRec | StopRec | PauseRec | StartFwd | StopFwd | BlkDTMF | UnblkDTMF | BlkMedia | UnblkMedia | PlayMedia | StopMedia | PlayDTMF | Stats | SlnMedia | UnslnMedia | Pub | SubReq | SubAns | Unsub | InjStart | InjStop | Conn | CLI | Trnsfm | Create | CrtAnsw | Mesh \n" "\n" "]\n" "totalpingcount\n" @@ -2561,6 +2609,8 @@ int main(void) { "0\n" "totalanswercount\n" "0\n" + "totalrollbackcount\n" + "0\n" "totaldeletecount\n" "0\n" "totalquerycount\n" @@ -2654,7 +2704,7 @@ int main(void) { "size\n" "16777208\n" "used\n" - "464\n" + "472\n" "}\n" "]\n" "}\n" @@ -2687,6 +2737,13 @@ int main(void) { "answers_ps_max 0 150\n" "answers_ps_avg 0 150\n" "answer_count 1 150\n" + "rollback_time_min 0.000000 150\n" + "rollback_time_max 0.000000 150\n" + "rollback_time_avg 0.000000 150\n" + "rollbacks_ps_min 0 150\n" + "rollbacks_ps_max 0 150\n" + "rollbacks_ps_avg 0 150\n" + "rollback_count 0 150\n" "delete_time_min 0.000000 150\n" "delete_time_max 0.000000 150\n" "delete_time_avg 0.000000 150\n" @@ -3171,6 +3228,14 @@ int main(void) { "3.200000\n" "avganswerdelay\n" "3.200000\n" + "Min/Max/Avg rollback processing delay\n" + "0.000000/0.000000/0.000000 sec\n" + "minrollbackdelay\n" + "0.000000\n" + "maxrollbackdelay\n" + "0.000000\n" + "avgrollbackdelay\n" + "0.000000\n" "Min/Max/Avg delete processing delay\n" "0.000000/0.000000/0.000000 sec\n" "mindeletedelay\n" @@ -3435,6 +3500,14 @@ int main(void) { "0\n" "avganswerrequestrate\n" "0\n" + "Min/Max/Avg rollback requests per second\n" + "0/0/0 per sec\n" + "minrollbackrequestrate\n" + "0\n" + "maxrollbackrequestrate\n" + "0\n" + "avgrollbackrequestrate\n" + "0\n" "Min/Max/Avg delete requests per second\n" "0/0/0 per sec\n" "mindeleterequestrate\n" @@ -3838,7 +3911,7 @@ int main(void) { "{\n" "proxies\n" "[\n" - " Proxy | Ping | Offer | Answer | Delete | Query | List | StartRec | StopRec | PauseRec | StartFwd | StopFwd | BlkDTMF | UnblkDTMF | BlkMedia | UnblkMedia | PlayMedia | StopMedia | PlayDTMF | Stats | SlnMedia | UnslnMedia | Pub | SubReq | SubAns | Unsub | InjStart | InjStop | Conn | CLI | Trnsfm | Create | CrtAnsw | Mesh \n" + " Proxy | Ping | Offer | Answer | Rollback | Delete | Query | List | StartRec | StopRec | PauseRec | StartFwd | StopFwd | BlkDTMF | UnblkDTMF | BlkMedia | UnblkMedia | PlayMedia | StopMedia | PlayDTMF | Stats | SlnMedia | UnslnMedia | Pub | SubReq | SubAns | Unsub | InjStart | InjStop | Conn | CLI | Trnsfm | Create | CrtAnsw | Mesh \n" "\n" "]\n" "totalpingcount\n" @@ -3847,6 +3920,8 @@ int main(void) { "0\n" "totalanswercount\n" "0\n" + "totalrollbackcount\n" + "0\n" "totaldeletecount\n" "0\n" "totalquerycount\n" @@ -3940,7 +4015,7 @@ int main(void) { "size\n" "16777208\n" "used\n" - "464\n" + "472\n" "}\n" "]\n" "}\n" @@ -3992,6 +4067,13 @@ int main(void) { "answers_ps_max 0 157\n" "answers_ps_avg 0 157\n" "answer_count 1 157\n" + "rollback_time_min 0.000000 157\n" + "rollback_time_max 0.000000 157\n" + "rollback_time_avg 0.000000 157\n" + "rollbacks_ps_min 0 157\n" + "rollbacks_ps_max 0 157\n" + "rollbacks_ps_avg 0 157\n" + "rollback_count 0 157\n" "delete_time_min 0.000000 157\n" "delete_time_max 0.000000 157\n" "delete_time_avg 0.000000 157\n" @@ -4476,6 +4558,14 @@ int main(void) { "0.000000\n" "avganswerdelay\n" "0.000000\n" + "Min/Max/Avg rollback processing delay\n" + "0.000000/0.000000/0.000000 sec\n" + "minrollbackdelay\n" + "0.000000\n" + "maxrollbackdelay\n" + "0.000000\n" + "avgrollbackdelay\n" + "0.000000\n" "Min/Max/Avg delete processing delay\n" "0.000000/0.000000/0.000000 sec\n" "mindeletedelay\n" @@ -4740,6 +4830,14 @@ int main(void) { "0\n" "avganswerrequestrate\n" "0\n" + "Min/Max/Avg rollback requests per second\n" + "0/0/0 per sec\n" + "minrollbackrequestrate\n" + "0\n" + "maxrollbackrequestrate\n" + "0\n" + "avgrollbackrequestrate\n" + "0\n" "Min/Max/Avg delete requests per second\n" "0/0/0 per sec\n" "mindeleterequestrate\n" @@ -5143,7 +5241,7 @@ int main(void) { "{\n" "proxies\n" "[\n" - " Proxy | Ping | Offer | Answer | Delete | Query | List | StartRec | StopRec | PauseRec | StartFwd | StopFwd | BlkDTMF | UnblkDTMF | BlkMedia | UnblkMedia | PlayMedia | StopMedia | PlayDTMF | Stats | SlnMedia | UnslnMedia | Pub | SubReq | SubAns | Unsub | InjStart | InjStop | Conn | CLI | Trnsfm | Create | CrtAnsw | Mesh \n" + " Proxy | Ping | Offer | Answer | Rollback | Delete | Query | List | StartRec | StopRec | PauseRec | StartFwd | StopFwd | BlkDTMF | UnblkDTMF | BlkMedia | UnblkMedia | PlayMedia | StopMedia | PlayDTMF | Stats | SlnMedia | UnslnMedia | Pub | SubReq | SubAns | Unsub | InjStart | InjStop | Conn | CLI | Trnsfm | Create | CrtAnsw | Mesh \n" "\n" "]\n" "totalpingcount\n" @@ -5152,6 +5250,8 @@ int main(void) { "0\n" "totalanswercount\n" "0\n" + "totalrollbackcount\n" + "0\n" "totaldeletecount\n" "0\n" "totalquerycount\n" @@ -5245,7 +5345,7 @@ int main(void) { "size\n" "16777208\n" "used\n" - "464\n" + "472\n" "}\n" "]\n" "}\n" @@ -5286,6 +5386,13 @@ int main(void) { "answers_ps_max 0 157\n" "answers_ps_avg 0 157\n" "answer_count 1 157\n" + "rollback_time_min 0.000000 157\n" + "rollback_time_max 0.000000 157\n" + "rollback_time_avg 0.000000 157\n" + "rollbacks_ps_min 0 157\n" + "rollbacks_ps_max 0 157\n" + "rollbacks_ps_avg 0 157\n" + "rollback_count 0 157\n" "delete_time_min 0.000000 157\n" "delete_time_max 0.000000 157\n" "delete_time_avg 0.000000 157\n" @@ -5770,6 +5877,14 @@ int main(void) { "0.000000\n" "avganswerdelay\n" "0.000000\n" + "Min/Max/Avg rollback processing delay\n" + "0.000000/0.000000/0.000000 sec\n" + "minrollbackdelay\n" + "0.000000\n" + "maxrollbackdelay\n" + "0.000000\n" + "avgrollbackdelay\n" + "0.000000\n" "Min/Max/Avg delete processing delay\n" "0.000000/0.000000/0.000000 sec\n" "mindeletedelay\n" @@ -6034,6 +6149,14 @@ int main(void) { "0\n" "avganswerrequestrate\n" "0\n" + "Min/Max/Avg rollback requests per second\n" + "0/0/0 per sec\n" + "minrollbackrequestrate\n" + "0\n" + "maxrollbackrequestrate\n" + "0\n" + "avgrollbackrequestrate\n" + "0\n" "Min/Max/Avg delete requests per second\n" "0/0/0 per sec\n" "mindeleterequestrate\n" @@ -6437,7 +6560,7 @@ int main(void) { "{\n" "proxies\n" "[\n" - " Proxy | Ping | Offer | Answer | Delete | Query | List | StartRec | StopRec | PauseRec | StartFwd | StopFwd | BlkDTMF | UnblkDTMF | BlkMedia | UnblkMedia | PlayMedia | StopMedia | PlayDTMF | Stats | SlnMedia | UnslnMedia | Pub | SubReq | SubAns | Unsub | InjStart | InjStop | Conn | CLI | Trnsfm | Create | CrtAnsw | Mesh \n" + " Proxy | Ping | Offer | Answer | Rollback | Delete | Query | List | StartRec | StopRec | PauseRec | StartFwd | StopFwd | BlkDTMF | UnblkDTMF | BlkMedia | UnblkMedia | PlayMedia | StopMedia | PlayDTMF | Stats | SlnMedia | UnslnMedia | Pub | SubReq | SubAns | Unsub | InjStart | InjStop | Conn | CLI | Trnsfm | Create | CrtAnsw | Mesh \n" "\n" "]\n" "totalpingcount\n" @@ -6446,6 +6569,8 @@ int main(void) { "0\n" "totalanswercount\n" "0\n" + "totalrollbackcount\n" + "0\n" "totaldeletecount\n" "0\n" "totalquerycount\n" @@ -6539,7 +6664,7 @@ int main(void) { "size\n" "16777208\n" "used\n" - "464\n" + "472\n" "}\n" "]\n" "}\n" @@ -6574,6 +6699,13 @@ int main(void) { "answers_ps_max 0 200\n" "answers_ps_avg 0 200\n" "answer_count 1 200\n" + "rollback_time_min 0.000000 200\n" + "rollback_time_max 0.000000 200\n" + "rollback_time_avg 0.000000 200\n" + "rollbacks_ps_min 0 200\n" + "rollbacks_ps_max 0 200\n" + "rollbacks_ps_avg 0 200\n" + "rollback_count 0 200\n" "delete_time_min 0.000000 200\n" "delete_time_max 0.000000 200\n" "delete_time_avg 0.000000 200\n" @@ -7058,6 +7190,14 @@ int main(void) { "0.000000\n" "avganswerdelay\n" "0.000000\n" + "Min/Max/Avg rollback processing delay\n" + "0.000000/0.000000/0.000000 sec\n" + "minrollbackdelay\n" + "0.000000\n" + "maxrollbackdelay\n" + "0.000000\n" + "avgrollbackdelay\n" + "0.000000\n" "Min/Max/Avg delete processing delay\n" "0.000000/0.000000/0.000000 sec\n" "mindeletedelay\n" @@ -7322,6 +7462,14 @@ int main(void) { "0\n" "avganswerrequestrate\n" "0\n" + "Min/Max/Avg rollback requests per second\n" + "0/0/0 per sec\n" + "minrollbackrequestrate\n" + "0\n" + "maxrollbackrequestrate\n" + "0\n" + "avgrollbackrequestrate\n" + "0\n" "Min/Max/Avg delete requests per second\n" "0/0/0 per sec\n" "mindeleterequestrate\n" @@ -7725,7 +7873,7 @@ int main(void) { "{\n" "proxies\n" "[\n" - " Proxy | Ping | Offer | Answer | Delete | Query | List | StartRec | StopRec | PauseRec | StartFwd | StopFwd | BlkDTMF | UnblkDTMF | BlkMedia | UnblkMedia | PlayMedia | StopMedia | PlayDTMF | Stats | SlnMedia | UnslnMedia | Pub | SubReq | SubAns | Unsub | InjStart | InjStop | Conn | CLI | Trnsfm | Create | CrtAnsw | Mesh \n" + " Proxy | Ping | Offer | Answer | Rollback | Delete | Query | List | StartRec | StopRec | PauseRec | StartFwd | StopFwd | BlkDTMF | UnblkDTMF | BlkMedia | UnblkMedia | PlayMedia | StopMedia | PlayDTMF | Stats | SlnMedia | UnslnMedia | Pub | SubReq | SubAns | Unsub | InjStart | InjStop | Conn | CLI | Trnsfm | Create | CrtAnsw | Mesh \n" "\n" "]\n" "totalpingcount\n" @@ -7734,6 +7882,8 @@ int main(void) { "0\n" "totalanswercount\n" "0\n" + "totalrollbackcount\n" + "0\n" "totaldeletecount\n" "0\n" "totalquerycount\n" @@ -7827,7 +7977,7 @@ int main(void) { "size\n" "16777208\n" "used\n" - "464\n" + "472\n" "}\n" "]\n" "}\n" @@ -7865,6 +8015,13 @@ int main(void) { "answers_ps_max 0 200\n" "answers_ps_avg 0 200\n" "answer_count 1 200\n" + "rollback_time_min 0.000000 200\n" + "rollback_time_max 0.000000 200\n" + "rollback_time_avg 0.000000 200\n" + "rollbacks_ps_min 0 200\n" + "rollbacks_ps_max 0 200\n" + "rollbacks_ps_avg 0 200\n" + "rollback_count 0 200\n" "delete_time_min 0.000000 200\n" "delete_time_max 0.000000 200\n" "delete_time_avg 0.000000 200\n" @@ -8349,6 +8506,14 @@ int main(void) { "0.000000\n" "avganswerdelay\n" "0.000000\n" + "Min/Max/Avg rollback processing delay\n" + "0.000000/0.000000/0.000000 sec\n" + "minrollbackdelay\n" + "0.000000\n" + "maxrollbackdelay\n" + "0.000000\n" + "avgrollbackdelay\n" + "0.000000\n" "Min/Max/Avg delete processing delay\n" "0.000000/0.000000/0.000000 sec\n" "mindeletedelay\n" @@ -8613,6 +8778,14 @@ int main(void) { "0\n" "avganswerrequestrate\n" "0\n" + "Min/Max/Avg rollback requests per second\n" + "0/0/0 per sec\n" + "minrollbackrequestrate\n" + "0\n" + "maxrollbackrequestrate\n" + "0\n" + "avgrollbackrequestrate\n" + "0\n" "Min/Max/Avg delete requests per second\n" "0/0/0 per sec\n" "mindeleterequestrate\n" @@ -9016,7 +9189,7 @@ int main(void) { "{\n" "proxies\n" "[\n" - " Proxy | Ping | Offer | Answer | Delete | Query | List | StartRec | StopRec | PauseRec | StartFwd | StopFwd | BlkDTMF | UnblkDTMF | BlkMedia | UnblkMedia | PlayMedia | StopMedia | PlayDTMF | Stats | SlnMedia | UnslnMedia | Pub | SubReq | SubAns | Unsub | InjStart | InjStop | Conn | CLI | Trnsfm | Create | CrtAnsw | Mesh \n" + " Proxy | Ping | Offer | Answer | Rollback | Delete | Query | List | StartRec | StopRec | PauseRec | StartFwd | StopFwd | BlkDTMF | UnblkDTMF | BlkMedia | UnblkMedia | PlayMedia | StopMedia | PlayDTMF | Stats | SlnMedia | UnslnMedia | Pub | SubReq | SubAns | Unsub | InjStart | InjStop | Conn | CLI | Trnsfm | Create | CrtAnsw | Mesh \n" "\n" "]\n" "totalpingcount\n" @@ -9025,6 +9198,8 @@ int main(void) { "0\n" "totalanswercount\n" "0\n" + "totalrollbackcount\n" + "0\n" "totaldeletecount\n" "0\n" "totalquerycount\n" @@ -9118,7 +9293,7 @@ int main(void) { "size\n" "16777208\n" "used\n" - "464\n" + "472\n" "}\n" "]\n" "}\n"